// RevokeAddrFS ... func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *pb.RevokeAddrFSRequest) (*pb.RevokeAddrFSResponse, error) { var err error srcAddr := "" // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // Validate Token _, err = s.validateToken(r.Token) if err != nil { log.Printf("%s REVOKE FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } // REVOKE an file system entry for the addr // delete /fs/FSID/addr addr AddrRef pKey := fmt.Sprintf("/fs/%s/addr", r.FSid) pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) _, err = s.gstore.Delete(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro) if store.IsNotFound(err) { log.Printf("%s REVOKE FAILED %s %s\n", srcAddr, r.FSid, r.Addr) return nil, errf(codes.NotFound, "%v", "Not Found") } // return Addr was revoked // Log Operation log.Printf("%s REVOKE SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr) return &pb.RevokeAddrFSResponse{Data: r.FSid}, nil }
// RevokeAddrFS ... func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *pb.RevokeAddrFSRequest) (*pb.RevokeAddrFSResponse, error) { var err error var acctID string var value []byte var fsRef FileSysRef srcAddr := "" // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // Validate Token acctID, err = s.validateToken(r.Token) if err != nil { log.Printf("%s REVOKE FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } // Validate Token/Account owns this file system // Read FileSysRef entry to determine if it exists pKey := fmt.Sprintf("/fs") pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(r.FSid)) _, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { log.Printf("%s REVOKE FAILED %s NOTFOUND", srcAddr, r.FSid) return nil, errf(codes.NotFound, "%v", "Not Found") } if err != nil { log.Printf("%s REVOKE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } err = json.Unmarshal(value, &fsRef) if err != nil { log.Printf("%s REVOKE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } if fsRef.AcctID != acctID { log.Printf("$s REVOKE FAILED %v ACCOUNT MISMATCH", r.FSid) return nil, errf(codes.FailedPrecondition, "%v", "Account Mismatch") } // REVOKE an file system entry for the addr // delete /fs/FSID/addr addr AddrRef pKey = fmt.Sprintf("/fs/%s/addr", r.FSid) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB = murmur3.Sum128([]byte(r.Addr)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) _, err = s.gstore.Delete(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro) if store.IsNotFound(err) { log.Printf("%s REVOKE FAILED %s %s\n", srcAddr, r.FSid, r.Addr) return nil, errf(codes.NotFound, "%v", "Not Found") } // return Addr was revoked // Log Operation log.Printf("%s REVOKE SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr) return &pb.RevokeAddrFSResponse{Data: r.FSid}, nil }
// GrantAddrFS ... func (s *FileSystemAPIServer) GrantAddrFS(ctx context.Context, r *fb.GrantAddrFSRequest) (*fb.GrantAddrFSResponse, error) { var status string var err error var acctData AcctPayLoad var fsData FileSysPayLoad var addrData AddrPayLoad var dataB []byte // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { fmt.Println(pr.Addr) } // getAcct data acctData, err = s.getAcct("/acct", r.Acctnum) if err != nil { log.Printf("Error %v on lookup for account %s", err, r.Acctnum) return nil, err } // validate token if acctData.Token != r.Token { return nil, errf(codes.PermissionDenied, "%s", "Invalid Token") } // getFS data fs := fmt.Sprintf("/acct/%s/fs", r.Acctnum) fsData, err = s.getFS(fs, r.FSid) if err != nil { log.Printf("Error %v on lookup for File system %s", err, r.Acctnum) return nil, err } if fsData.Status == "active" { log.Println("FileSystem is active") } // write out the ip address parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid) childKey := r.Addr parentKeyA, parentKeyB := murmur3.Sum128([]byte(parentKey)) childKeyA, childKeyB := murmur3.Sum128([]byte(childKey)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) addrData.Addr = r.Addr dataB, err = json.Marshal(addrData) if err != nil { log.Printf("Marshal Error: %v\n...", err) return nil, errf(codes.Internal, "%v", err) } _, err = s.fsws.gstore.Write(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro, dataB) if err != nil { log.Printf("Write Error: %v", err) return nil, errf(codes.Internal, "%v", err) } // DO stuff status = fmt.Sprintf("addr %s for filesystem %s with account id %s was granted", r.Addr, r.FSid, r.Acctnum) return &fb.GrantAddrFSResponse{Status: status}, nil }
func (o *StoreComms) DeleteGroupItemTS(ctx context.Context, key, childKey []byte, tsm int64) error { keyA, keyB := murmur3.Sum128(key) childKeyA, childKeyB := murmur3.Sum128(childKey) oldTimestampMicro, err := o.gstore.Delete(ctx, keyA, keyB, childKeyA, childKeyB, tsm) if err != nil { return err } if oldTimestampMicro >= tsm { return ErrStoreHasNewerValue } return nil }
// writeGStore ... func (fsws *FileSystemWS) writeGStore(g string, m string, p []byte) (string, error) { // prepare groupVal and memberVal log.Println("Starting a Write to the Group Store") keyA, keyB := murmur3.Sum128([]byte(g)) childKeyA, childKeyB := murmur3.Sum128([]byte(m)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) newTimestampMicro, err := fsws.gstore.Write(context.Background(), keyA, keyB, childKeyA, childKeyB, timestampMicro, p) if err != nil { return "", err } log.Println("Successfully wrote something to the Group Store") return fmt.Sprintf("TSM: %d", newTimestampMicro), nil }
// lookupAccount ... func (fsws *FileSystemWS) getGStore(g string, m string) (string, error) { log.Println("Starting a Read from the Group Store") keyA, keyB := murmur3.Sum128([]byte(g)) childKeyA, childKeyB := murmur3.Sum128([]byte(m)) _, value, err := fsws.gstore.Read(context.Background(), keyA, keyB, childKeyA, childKeyB, nil) if store.IsNotFound(err) { log.Printf(" Not Found Key: %d, %d ChildKey: %d, %d", keyA, keyB, childKeyA, childKeyB) return "", nil } else if err != nil { return "", err } log.Println("Successfully read an item from the Group Store") return fmt.Sprintf("%s", value), nil }
// ShowFS ... func (s *FileSystemAPIServer) ShowFS(ctx context.Context, r *fb.ShowFSRequest) (*fb.ShowFSResponse, error) { var status string var acctData AcctPayLoad var fsData FileSysPayLoad var fsDataB []byte var err error // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { fmt.Println(pr.Addr) } // getAcct data acctData, err = s.getAcct("/acct", r.Acctnum) if err != nil { log.Printf("Error %v on lookup for account %s", err, r.Acctnum) return nil, err } // validate token if acctData.Token != r.Token { return nil, errf(codes.PermissionDenied, "%s", "Invalid Token") } pKey := fmt.Sprintf("/acct/%s/fs", r.Acctnum) pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(r.FSid)) _, fsDataB, err = s.fsws.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { return nil, errf(codes.NotFound, "%v", "File System Not Found") } if err != nil { return nil, errf(codes.Internal, "%s", err) } err = json.Unmarshal(fsDataB, &fsData) if err != nil { return nil, errf(codes.Internal, "%s", err) } // Get list of Addr fsData.Addr, err = s.addrList(fsData.ID) if err != nil { return nil, errf(codes.Internal, "%s", err) } fsDataB, err = json.Marshal(&fsData) if err != nil { return nil, errf(codes.Internal, "%s", err) } // Prep things to return status = "OK" return &fb.ShowFSResponse{Payload: string(fsDataB), Status: status}, nil }
// Helper methods to get data from value and group store func (o *StoreComms) ReadValue(ctx context.Context, id []byte) ([]byte, error) { // TODO: You might want to make this whole area pass in reusable []byte to // lessen gc pressure. keyA, keyB := murmur3.Sum128(id) _, v, err := o.vstore.Read(ctx, keyA, keyB, nil) return v, err }
func (ktbf *groupKTBloomFilter) mayHave(keyA uint64, keyB uint64, childKeyA uint64, childKeyB uint64, timestamp uint64) bool { scratch := ktbf.scratch binary.BigEndian.PutUint64(scratch[4:], keyA) binary.BigEndian.PutUint64(scratch[12:], keyB) binary.BigEndian.PutUint64(scratch[20:], childKeyA) binary.BigEndian.PutUint64(scratch[28:], childKeyB) binary.BigEndian.PutUint64(scratch[36:], timestamp) for i := ktbf.kDiv4; i > 0; i-- { binary.BigEndian.PutUint32(scratch, ktbf.salt|i) h1, h2 := murmur3.Sum128(scratch) bit := uint32(h1>>32) % ktbf.m if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 { return false } bit = uint32(h1&0xffffffff) % ktbf.m if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 { return false } bit = uint32(h2>>32) % ktbf.m if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 { return false } bit = uint32(h2&0xffffffff) % ktbf.m if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 { return false } } return true }
func (ktbf *groupKTBloomFilter) add(keyA uint64, keyB uint64, childKeyA uint64, childKeyB uint64, timestamp uint64) { // CONSIDER: There are optimization opportunities here as the keys can be // considered to already have good bit distribution and using a hashing // function to mix-in timestamp, salt, and i instead of redoing the whole // hash each time would be good to test and benchmark. scratch := ktbf.scratch binary.BigEndian.PutUint64(scratch[4:], keyA) binary.BigEndian.PutUint64(scratch[12:], keyB) binary.BigEndian.PutUint64(scratch[20:], childKeyA) binary.BigEndian.PutUint64(scratch[28:], childKeyB) binary.BigEndian.PutUint64(scratch[36:], timestamp) for i := ktbf.kDiv4; i > 0; i-- { binary.BigEndian.PutUint32(scratch, ktbf.salt|i) h1, h2 := murmur3.Sum128(scratch) bit := uint32(h1>>32) % ktbf.m ktbf.bits[bit/8] |= 1 << (bit % 8) bit = uint32(h1&0xffffffff) % ktbf.m ktbf.bits[bit/8] |= 1 << (bit % 8) bit = uint32(h2>>32) % ktbf.m ktbf.bits[bit/8] |= 1 << (bit % 8) bit = uint32(h2&0xffffffff) % ktbf.m ktbf.bits[bit/8] |= 1 << (bit % 8) } }
func (o *StoreComms) ReadGroup(ctx context.Context, key []byte) ([]store.ReadGroupItem, error) { keyA, keyB := murmur3.Sum128(key) items, err := o.gstore.ReadGroup(ctx, keyA, keyB) if err != nil { return nil, err } return items, nil }
func (o *StoreComms) DeleteValueTS(ctx context.Context, id []byte, tsm int64) error { keyA, keyB := murmur3.Sum128(id) oldTimestampMicro, err := o.vstore.Delete(ctx, keyA, keyB, tsm) if oldTimestampMicro >= tsm { return ErrStoreHasNewerValue } return err }
// HashBytes will return the hash for the provided byte slice. func HashBytes(b []byte) []byte { h1, h2 := murmur3.Sum128(b) return []byte{ byte(h1 >> 56), byte(h1 >> 48), byte(h1 >> 40), byte(h1 >> 32), byte(h1 >> 24), byte(h1 >> 16), byte(h1 >> 8), byte(h1), byte(h2 >> 56), byte(h2 >> 48), byte(h2 >> 40), byte(h2 >> 32), byte(h2 >> 24), byte(h2 >> 16), byte(h2 >> 8), byte(h2), } }
// validateToken ... func (s *FileSystemAPIServer) validateToken(t string) (string, error) { var tData TokenRef var aData AcctPayLoad var tDataByte []byte var aDataByte []byte var err error // Read Token pKeyA, pKeyB := murmur3.Sum128([]byte("/token")) cKeyA, cKeyB := murmur3.Sum128([]byte(t)) _, tDataByte, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { return "", errors.New("Not Found") } err = json.Unmarshal(tDataByte, &tData) if err != nil { log.Printf("TOKEN FAILED %v\n", err) return "", err } // Read Account pKeyA, pKeyB = murmur3.Sum128([]byte("/acct")) cKeyA, cKeyB = murmur3.Sum128([]byte(tData.AcctID)) _, aDataByte, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { return "", errors.New("Not Found") } err = json.Unmarshal(aDataByte, &aData) if err != nil { log.Printf("TOKEN FAILED %v\n", err) return "", err } if tData.TokenID != aData.Token { // Log Failed Operation log.Printf("TOKEN FAIL %s\n", t) return "", errors.New("Invalid Token") } // Return Account UUID // Log Operation log.Printf("TOKEN SUCCESS %s\n", tData.AcctID) return tData.AcctID, nil }
func VSTests() { vsconfigs := make([]ValueClientConfig, *clients) var wg sync.WaitGroup for w := 0; w < *clients; w++ { vsconfigs[w].addr = *vsServer vsconfigs[w].id = w vsconfigs[w].count = perClient vsconfigs[w].value = &value vsconfigs[w].wg = &wg vsconfigs[w].wm = make([]*vp.WriteRequest, perClient) vsconfigs[w].rm = make([]*vp.ReadRequest, perClient) for k := 0; k < perClient; k++ { vsconfigs[w].wm[k] = &vp.WriteRequest{} vsconfigs[w].rm[k] = &vp.ReadRequest{} vsconfigs[w].wm[k].KeyA, vsconfigs[w].wm[k].KeyB = murmur3.Sum128([]byte(fmt.Sprintf("somethingtestkey%d-%d", vsconfigs[w].id, k))) vsconfigs[w].wm[k].TimestampMicro = brimtime.TimeToUnixMicro(time.Now()) vsconfigs[w].rm[k].KeyA = vsconfigs[w].wm[k].KeyA vsconfigs[w].rm[k].KeyB = vsconfigs[w].wm[k].KeyB } } log.Println("ValueStore Key/hash generation complete. Spawning tests.") // ValueStore Tests if *vsWriteTest { t := time.Now() for w := 0; w < *clients; w++ { wg.Add(1) if *streamTest { go ValueStreamWrite(&vsconfigs[w]) } else { go ValueWrite(&vsconfigs[w]) } } wg.Wait() log.Println("Issued", *clients*perClient, "VS WRITES") ts := time.Since(t).Seconds() log.Println("Total run time was:", ts, "seconds") log.Printf("Per second: %.2f\n", float64(*clients*perClient)/ts) } if *vsReadTest { t := time.Now() for w := 0; w < *clients; w++ { wg.Add(1) if *streamTest { go ValueStreamRead(&vsconfigs[w]) } else { go ValueRead(&vsconfigs[w]) } } wg.Wait() log.Println("Issued", *clients*perClient, "VS READS") ts := time.Since(t).Seconds() log.Println("Total run time was:", ts, "seconds") log.Printf("Per second: %.2f\n", float64(*clients*perClient)/ts) } }
// GrantAddrFS ... func (s *FileSystemAPIServer) GrantAddrFS(ctx context.Context, r *pb.GrantAddrFSRequest) (*pb.GrantAddrFSResponse, error) { var err error var addrData AddrRef var addrByte []byte srcAddr := "" // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // validate token _, err = s.validateToken(r.Token) if err != nil { log.Printf("%s GRANT FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } // GRANT an file system entry for the addr // write /fs/FSID/addr addr AddrRef pKey := fmt.Sprintf("/fs/%s/addr", r.FSid) pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) addrData.Addr = r.Addr addrData.FSID = r.FSid addrByte, err = json.Marshal(addrData) if err != nil { log.Printf("%s GRANT FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } _, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, addrByte) if err != nil { log.Printf("%s GRANT FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } // return Addr was Granted // Log Operation log.Printf("%s GRANT SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr) return &pb.GrantAddrFSResponse{Data: r.FSid}, nil }
func (o *StoreComms) WriteValue(ctx context.Context, id, data []byte) error { keyA, keyB := murmur3.Sum128(id) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) oldTimestampMicro, err := o.vstore.Write(ctx, keyA, keyB, timestampMicro, data) if err != nil { return err } if oldTimestampMicro >= timestampMicro { return ErrStoreHasNewerValue } return nil }
// LookupAddrFS ... func (s *FileSystemAPIServer) LookupAddrFS(ctx context.Context, r *fb.LookupAddrFSRequest) (*fb.LookupAddrFSResponse, error) { var err error var status string // Check if IP is assigned to the file system parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid) pKeyA, pKeyB := murmur3.Sum128([]byte(parentKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr)) _, _, err = s.fsws.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { log.Printf("/fs/%s/addr/%s did not exist to delete", r.FSid, r.Addr) return nil, errf(codes.NotFound, "%s", "Addr not found") } else if err != nil { return nil, errf(codes.Internal, "%s", err) } // // Verify Account is activea status = "OK" return &fb.LookupAddrFSResponse{Status: status}, nil }
// RevokeAddrFS ... func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *fb.RevokeAddrFSRequest) (*fb.RevokeAddrFSResponse, error) { var status string var err error var acctData AcctPayLoad // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { fmt.Println(pr.Addr) } // getAcct data acctData, err = s.getAcct("/acct", r.Acctnum) if err != nil { log.Printf("Error %v on lookup for account %s", err, r.Acctnum) return nil, errf(codes.NotFound, "%v", err) } // validate token if acctData.Token != r.Token { return nil, errf(codes.PermissionDenied, "%s", "Invalid Token") } parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid) childKey := r.Addr parentKeyA, parentKeyB := murmur3.Sum128([]byte(parentKey)) childKeyA, childKeyB := murmur3.Sum128([]byte(childKey)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) // Delete addr _, err = s.fsws.gstore.Delete(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro) if store.IsNotFound(err) { log.Printf("/fs/%s/addr/%s did not exist to delete", r.FSid, r.Addr) return nil, errf(codes.NotFound, "%s", "Addr not found") } else if err != nil { return nil, errf(codes.Internal, "%s", err) } // DO stuff status = fmt.Sprintf("addr %s for filesystem %s with account id %s was revoked", r.Addr, r.FSid, r.Acctnum) return &fb.RevokeAddrFSResponse{Status: status}, nil }
func EtagMatch(w http.ResponseWriter, r *http.Request, contents []byte) bool { tHash := time.Now() h1, h2 := murmur3.Sum128(contents) etag := fmt.Sprintf(`"%x%x"`, h1, h2) glog.V(3).Infof("PERF: murmurm3 generation time: %v\n", time.Now().Sub(tHash)) ifNoneMatch := r.Header.Get("If-None-Match") if ifNoneMatch == etag { w.WriteHeader(304) return true } w.Header().Set("Etag", etag) return false }
func (self *Hash) Extrude(result []byte) { h1, h2 := murmur3.Sum128((*bytes.Buffer)(self).Bytes()) result[0] = byte(h1 >> 56) result[1] = byte(h1 >> 48) result[2] = byte(h1 >> 40) result[3] = byte(h1 >> 32) result[4] = byte(h1 >> 24) result[5] = byte(h1 >> 16) result[6] = byte(h1 >> 8) result[7] = byte(h1) result[8] = byte(h2 >> 56) result[9] = byte(h2 >> 48) result[10] = byte(h2 >> 40) result[11] = byte(h2 >> 32) result[12] = byte(h2 >> 24) result[13] = byte(h2 >> 16) result[14] = byte(h2 >> 8) result[15] = byte(h2) }
// grpc Group Store functions // lookupAccount ... func (fsws *FileSystemWS) readGroupGStore(g string) (string, error) { log.Println("Starting a Group Store Lookup") keyA, keyB := murmur3.Sum128([]byte(g)) items, err := fsws.gstore.ReadGroup(context.Background(), keyA, keyB) if store.IsNotFound(err) { log.Printf("Not Found: %s", g) return "", nil } else if err != nil { return "", err } // Build a list of accounts. log.Println("Build a list of file systems") m := make([]string, len(items)) for k, v := range items { m[k] = fmt.Sprintf("%s", v.Value) } log.Println("Returning a list of file systems") return fmt.Sprintf(strings.Join(m, "|")), nil }
// addrList ... func (s *FileSystemAPIServer) addrList(fsid string) ([]string, error) { var ap AddrPayLoad pKey := fmt.Sprintf("/fs/%s/addr", fsid) pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) items, err := s.fsws.gstore.ReadGroup(context.Background(), pKeyA, pKeyB) if store.IsNotFound(err) { return nil, nil } if err != nil { return nil, err } l := make([]string, len(items)) for k, v := range items { err = json.Unmarshal(v.Value, &ap) if err != nil { return nil, err } l[k] = ap.Addr } return l, nil }
func (o *StoreComms) ReadGroupItem(ctx context.Context, key, childKey []byte) ([]byte, error) { childKeyA, childKeyB := murmur3.Sum128(childKey) return o.ReadGroupItemByKey(ctx, key, childKeyA, childKeyB) }
// CreateFS ... func (s *FileSystemAPIServer) CreateFS(ctx context.Context, r *pb.CreateFSRequest) (*pb.CreateFSResponse, error) { var err error var acctID string srcAddr := "" var fsRef FileSysRef var fsRefByte []byte var fsSysAttr FileSysAttr var fsSysAttrByte []byte // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // Validate Token acctID, err = s.validateToken(r.Token) if err != nil { log.Printf("%s CREATE FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } fsID := uuid.NewV4().String() timestampMicro := brimtime.TimeToUnixMicro(time.Now()) // Write file system reference entries. // write /fs FSID FileSysRef pKey := "/fs" pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(fsID)) fsRef.AcctID = acctID fsRef.FSID = fsID fsRefByte, err = json.Marshal(fsRef) if err != nil { log.Printf("%s CREATE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } _, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsRefByte) if err != nil { log.Printf("%s CREATE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } // write /acct/acctID FSID FileSysRef pKey = fmt.Sprintf("/acct/%s", acctID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) _, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsRefByte) if err != nil { log.Printf("%s CREATE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } // Write file system attributes // write /fs/FSID name FileSysAttr pKey = fmt.Sprintf("/fs/%s", fsID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB = murmur3.Sum128([]byte("name")) fsSysAttr.Attr = "name" fsSysAttr.Value = r.FSName fsSysAttr.FSID = fsID fsSysAttrByte, err = json.Marshal(fsSysAttr) if err != nil { log.Printf("%s CREATE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } _, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsSysAttrByte) if err != nil { log.Printf("%s CREATE FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } // Return File System UUID // Log Operation log.Printf("%s CREATE SUCCESS %s\n", srcAddr, fsID) return &pb.CreateFSResponse{Data: fsID}, nil }
func (o *StoreComms) ReadGroupItemByKey(ctx context.Context, key []byte, childKeyA, childKeyB uint64) ([]byte, error) { keyA, keyB := murmur3.Sum128(key) _, v, err := o.gstore.Read(ctx, keyA, keyB, childKeyA, childKeyB, nil) return v, err }
// ListFS ... func (s *FileSystemAPIServer) ListFS(ctx context.Context, r *pb.ListFSRequest) (*pb.ListFSResponse, error) { srcAddr := "" // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // Validate Token acctID, err := s.validateToken(r.Token) if err != nil { log.Printf("%s LIST FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } var value []byte var fsRef FileSysRef var addrData AddrRef var fsAttrData FileSysAttr var aList []string // Read Group /acct/acctID _ FileSysRef pKey := fmt.Sprintf("/acct/%s", acctID) pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) list, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB) if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fsList := make([]FileSysMeta, len(list)) for k, v := range list { clear(&fsRef) clear(&addrData) clear(&fsAttrData) clear(&aList) err = json.Unmarshal(v.Value, &fsRef) if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fsList[k].AcctID = acctID fsList[k].ID = fsRef.FSID // Get File System Name pKey = fmt.Sprintf("/fs/%s", fsList[k].ID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte("name")) _, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { log.Printf("%s LIST FAILED %s NAMENOTFOUND", srcAddr, fsList[k].ID) return nil, errf(codes.NotFound, "%v", "File System Name Not Found") } if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } err = json.Unmarshal(value, &fsAttrData) if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fsList[k].Name = fsAttrData.Value // Get List of addrs pKey = fmt.Sprintf("/fs/%s/addr", fsList[k].ID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) items, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB) if !store.IsNotFound(err) { // No addr granted aList = make([]string, len(items)) for sk, sv := range items { err = json.Unmarshal(sv.Value, &addrData) if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } aList[sk] = addrData.Addr } } if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fsList[k].Addr = aList } // Return a File System List fsListJSON, jerr := json.Marshal(&fsList) if jerr != nil { return nil, errf(codes.Internal, "%s", jerr) } // Log Operation log.Printf("%s LIST SUCCESS %s\n", srcAddr, acctID) return &pb.ListFSResponse{Data: string(fsListJSON)}, nil }
// ShowFS ... func (s *FileSystemAPIServer) ShowFS(ctx context.Context, r *pb.ShowFSRequest) (*pb.ShowFSResponse, error) { var err error var acctID string srcAddr := "" // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { srcAddr = pr.Addr.String() } // Validate Token acctID, err = s.validateToken(r.Token) if err != nil { log.Printf("%s SHOW FAILED %s\n", srcAddr, "PermissionDenied") return nil, errf(codes.PermissionDenied, "%v", "Invalid Token") } var fs FileSysMeta var value []byte var fsRef FileSysRef var addrData AddrRef var fsAttrData FileSysAttr var aList []string fs.ID = r.FSid // Read FileSysRef entry to determine if it exists pKey := fmt.Sprintf("/fs") pKeyA, pKeyB := murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB := murmur3.Sum128([]byte(fs.ID)) _, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { log.Printf("%s SHOW FAILED %s NOTFOUND", srcAddr, r.FSid) return nil, errf(codes.NotFound, "%v", "Not Found") } if err != nil { log.Printf("%s SHOW FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } err = json.Unmarshal(value, &fsRef) if err != nil { log.Printf("%s SHOW FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } // Validate Token/Account own the file system if fsRef.AcctID != acctID { log.Printf("$s SHOW FAILED %v ACCOUNT MISMATCH", fs.ID) return nil, errf(codes.FailedPrecondition, "%v", "Account Mismatch") } fs.AcctID = fsRef.AcctID // Read the file system attributes // group-lookup /fs FSID // Iterate over all the atributes pKey = fmt.Sprintf("/fs/%s", fs.ID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) cKeyA, cKeyB = murmur3.Sum128([]byte("name")) _, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil) if store.IsNotFound(err) { log.Printf("%s SHOW FAILED %s NAMENOTFOUND", srcAddr, r.FSid) return nil, errf(codes.NotFound, "%v", "File System Name Not Found") } if err != nil { log.Printf("%s SHOW FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } err = json.Unmarshal(value, &fsAttrData) if err != nil { log.Printf("%s SHOW FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fs.Name = fsAttrData.Value // Read list of granted ip addresses // group-lookup printf("/fs/%s/addr", FSID) pKey = fmt.Sprintf("/fs/%s/addr", fs.ID) pKeyA, pKeyB = murmur3.Sum128([]byte(pKey)) items, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB) if !store.IsNotFound(err) { // No addr granted aList = make([]string, len(items)) for k, v := range items { err = json.Unmarshal(v.Value, &addrData) if err != nil { log.Printf("%s LIST FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } aList[k] = addrData.Addr } } if err != nil { log.Printf("%s SHOW FAILED %v\n", srcAddr, err) return nil, errf(codes.Internal, "%v", err) } fs.Addr = aList // Return File System fsJSON, jerr := json.Marshal(&fs) if jerr != nil { return nil, errf(codes.Internal, "%s", jerr) } // Log Operation log.Printf("%s SHOW SUCCESS %s\n", srcAddr, r.FSid) return &pb.ShowFSResponse{Data: string(fsJSON)}, nil }
// CreateFS ... func (s *FileSystemAPIServer) CreateFS(ctx context.Context, r *fb.CreateFSRequest) (*fb.CreateFSResponse, error) { var status string var result string var acctData AcctPayLoad var newFS FileSysPayLoad var acctRef AcctRefPayload var acctRefB []byte var err error var dataB []byte // Get incomming ip pr, ok := peer.FromContext(ctx) if ok { fmt.Println(pr.Addr) } // getAcct data acctData, err = s.getAcct("/acct", r.Acctnum) if err != nil { log.Printf("Error %v on lookup for account %s", err, r.Acctnum) return nil, err } // validate token if acctData.Token != r.Token { return nil, errf(codes.PermissionDenied, "%s", "Invalid Token") } // Check for to see if file system name exists fs := fmt.Sprintf("/acct/%s/fs", acctData.ID) err = s.dupNameCheck(fs, r.FSName) if err != nil { log.Printf("Precondition Failed: %v\n...", err) return nil, errf(codes.FailedPrecondition, "%v", err) } // File system values parentKey := fmt.Sprintf("/acct/%s/fs", r.Acctnum) childKey := uuid.NewV4().String() newFS.ID = childKey newFS.AcctID = r.Acctnum newFS.Name = r.FSName newFS.SizeInBytes = 107374182400 newFS.Status = "active" newFS.CreateDate = time.Now().Unix() newFS.DeleteDate = 0 //write file system dataB, err = json.Marshal(newFS) if err != nil { log.Printf("Marshal Error: %v\n...", err) return nil, errf(codes.Internal, "%v", err) } _, err = s.fsws.writeGStore(parentKey, childKey, dataB) if err != nil { log.Printf("Write Error: %v", err) return nil, errf(codes.Internal, "%v", err) } // Write special filesystem look up entry // "/fs" "[file system uuid]" {"id": "[filesystem uuid]", "acctid": "[account uuid]"} parentKeyA, parentKeyB := murmur3.Sum128([]byte("/fs")) childKeyA, childKeyB := murmur3.Sum128([]byte(newFS.ID)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) acctRef.ID = newFS.ID acctRef.AcctID = r.Acctnum acctRefB, err = json.Marshal(acctRef) if err != nil { log.Printf("Marshal Error: %v\n...", err) return nil, errf(codes.Internal, "%v", err) } _, err = s.fsws.gstore.Write(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro, acctRefB) if err != nil { log.Printf("Write Error: %v", err) return nil, errf(codes.Internal, "%v", err) } // Prep reults to return status = "OK" result = fmt.Sprintf("File System %s was created from Account %s", childKey, r.Acctnum) return &fb.CreateFSResponse{Payload: result, Status: status}, nil }
func (c *Client) parseValueCmd(line string) (string, error) { if c.vconn == nil { err := c.getValueClient() if err != nil { return "", err } } split := strings.SplitN(line, " ", 2) cmd := split[0] if len(split) != 2 { if cmd == "exit" { return "", fmt.Errorf("Exiting..") } if cmd == "help" { return c.printHelp(), nil } return c.printHelp(), nil } args := split[1] switch cmd { case "write": sarg := strings.SplitN(args, " ", 2) if len(sarg) < 2 { return fmt.Sprintf("write needs key and value: `write somekey some value thing here`"), nil } keyA, keyB := murmur3.Sum128([]byte(sarg[0])) value := []byte(sarg[1]) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) oldTimestampMicro, err := c.vstore.Write(context.Background(), keyA, keyB, timestampMicro, value) if err != nil { return "", err } return fmt.Sprintf("WRITE TIMESTAMPMICRO: %d\nPREVIOUS TIMESTAMPMICRO: %d", timestampMicro, oldTimestampMicro), nil case "read": keyA, keyB := murmur3.Sum128([]byte(args)) timestampMicro, value, err := c.vstore.Read(context.Background(), keyA, keyB, nil) if store.IsNotFound(err) { return fmt.Sprintf("not found"), nil } else if err != nil { return "", err } return fmt.Sprintf("TIMESTAMPMICRO: %d\nVALUE: %s", timestampMicro, value), nil case "delete": keyA, keyB := murmur3.Sum128([]byte(args)) timestampMicro := brimtime.TimeToUnixMicro(time.Now()) oldTimestampMicro, err := c.vstore.Delete(context.Background(), keyA, keyB, timestampMicro) if err != nil { return "", err } return fmt.Sprintf("TIMESTAMPMICRO: %d\nOLD TIMESTAMPMICRO: %d", timestampMicro, oldTimestampMicro), nil case "lookup": keyA, keyB := murmur3.Sum128([]byte(args)) timestampMicro, length, err := c.vstore.Lookup(context.Background(), keyA, keyB) if store.IsNotFound(err) { return fmt.Sprintf("not found"), nil } else if err != nil { return "", err } return fmt.Sprintf("TIMESTAMPMICRO: %d\nLENGTH: %d", timestampMicro, length), nil case "mode": if args == "value" { return fmt.Sprintf("Already in value store mode"), nil } if args == "group" { c.gmode = true return fmt.Sprintf("Switched to group mode"), nil } return fmt.Sprintf("Valid modes are: value | group"), nil case "exit": log.Println("exit") return "", fmt.Errorf("Exiting..") } return c.printHelp(), nil }