Exemple #1
0
// RevokeAddrFS ...
func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *pb.RevokeAddrFSRequest) (*pb.RevokeAddrFSResponse, error) {
	var err error
	srcAddr := ""

	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}
	// Validate Token
	_, err = s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s REVOKE FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}

	// REVOKE an file system entry for the addr
	// 		delete /fs/FSID/addr			addr						AddrRef
	pKey := fmt.Sprintf("/fs/%s/addr", r.FSid)
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	_, err = s.gstore.Delete(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro)
	if store.IsNotFound(err) {
		log.Printf("%s REVOKE FAILED %s %s\n", srcAddr, r.FSid, r.Addr)
		return nil, errf(codes.NotFound, "%v", "Not Found")
	}

	// return Addr was revoked
	// Log Operation
	log.Printf("%s REVOKE SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr)
	return &pb.RevokeAddrFSResponse{Data: r.FSid}, nil
}
Exemple #2
0
// RevokeAddrFS ...
func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *pb.RevokeAddrFSRequest) (*pb.RevokeAddrFSResponse, error) {
	var err error
	var acctID string
	var value []byte
	var fsRef FileSysRef
	srcAddr := ""

	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}
	// Validate Token
	acctID, err = s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s REVOKE FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}
	// Validate Token/Account owns this file system
	// Read FileSysRef entry to determine if it exists
	pKey := fmt.Sprintf("/fs")
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(r.FSid))
	_, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		log.Printf("%s REVOKE FAILED %s NOTFOUND", srcAddr, r.FSid)
		return nil, errf(codes.NotFound, "%v", "Not Found")
	}
	if err != nil {
		log.Printf("%s REVOKE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	err = json.Unmarshal(value, &fsRef)
	if err != nil {
		log.Printf("%s REVOKE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	if fsRef.AcctID != acctID {
		log.Printf("$s REVOKE FAILED %v ACCOUNT MISMATCH", r.FSid)
		return nil, errf(codes.FailedPrecondition, "%v", "Account Mismatch")
	}

	// REVOKE an file system entry for the addr
	// 		delete /fs/FSID/addr			addr						AddrRef
	pKey = fmt.Sprintf("/fs/%s/addr", r.FSid)
	pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB = murmur3.Sum128([]byte(r.Addr))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	_, err = s.gstore.Delete(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro)
	if store.IsNotFound(err) {
		log.Printf("%s REVOKE FAILED %s %s\n", srcAddr, r.FSid, r.Addr)
		return nil, errf(codes.NotFound, "%v", "Not Found")
	}

	// return Addr was revoked
	// Log Operation
	log.Printf("%s REVOKE SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr)
	return &pb.RevokeAddrFSResponse{Data: r.FSid}, nil
}
Exemple #3
0
// GrantAddrFS ...
func (s *FileSystemAPIServer) GrantAddrFS(ctx context.Context, r *fb.GrantAddrFSRequest) (*fb.GrantAddrFSResponse, error) {
	var status string
	var err error
	var acctData AcctPayLoad
	var fsData FileSysPayLoad
	var addrData AddrPayLoad
	var dataB []byte
	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		fmt.Println(pr.Addr)
	}
	// getAcct data
	acctData, err = s.getAcct("/acct", r.Acctnum)
	if err != nil {
		log.Printf("Error %v on lookup for account %s", err, r.Acctnum)
		return nil, err
	}
	// validate token
	if acctData.Token != r.Token {
		return nil, errf(codes.PermissionDenied, "%s", "Invalid Token")
	}
	// getFS data
	fs := fmt.Sprintf("/acct/%s/fs", r.Acctnum)
	fsData, err = s.getFS(fs, r.FSid)
	if err != nil {
		log.Printf("Error %v on lookup for File system %s", err, r.Acctnum)
		return nil, err
	}
	if fsData.Status == "active" {
		log.Println("FileSystem is active")
	}
	// write out the ip address
	parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid)
	childKey := r.Addr

	parentKeyA, parentKeyB := murmur3.Sum128([]byte(parentKey))
	childKeyA, childKeyB := murmur3.Sum128([]byte(childKey))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	addrData.Addr = r.Addr
	dataB, err = json.Marshal(addrData)
	if err != nil {
		log.Printf("Marshal Error: %v\n...", err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.fsws.gstore.Write(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro, dataB)
	if err != nil {
		log.Printf("Write Error: %v", err)
		return nil, errf(codes.Internal, "%v", err)
	}

	// DO stuff
	status = fmt.Sprintf("addr %s for filesystem %s with account id %s was granted", r.Addr, r.FSid, r.Acctnum)
	return &fb.GrantAddrFSResponse{Status: status}, nil
}
func (o *StoreComms) DeleteGroupItemTS(ctx context.Context, key, childKey []byte, tsm int64) error {
	keyA, keyB := murmur3.Sum128(key)
	childKeyA, childKeyB := murmur3.Sum128(childKey)
	oldTimestampMicro, err := o.gstore.Delete(ctx, keyA, keyB, childKeyA, childKeyB, tsm)
	if err != nil {
		return err
	}
	if oldTimestampMicro >= tsm {
		return ErrStoreHasNewerValue
	}
	return nil
}
Exemple #5
0
// writeGStore ...
func (fsws *FileSystemWS) writeGStore(g string, m string, p []byte) (string, error) {
	// prepare groupVal and memberVal
	log.Println("Starting a Write to the Group Store")
	keyA, keyB := murmur3.Sum128([]byte(g))
	childKeyA, childKeyB := murmur3.Sum128([]byte(m))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	newTimestampMicro, err := fsws.gstore.Write(context.Background(), keyA, keyB, childKeyA, childKeyB, timestampMicro, p)
	if err != nil {
		return "", err
	}
	log.Println("Successfully wrote something to the Group Store")
	return fmt.Sprintf("TSM: %d", newTimestampMicro), nil
}
Exemple #6
0
// lookupAccount ...
func (fsws *FileSystemWS) getGStore(g string, m string) (string, error) {
	log.Println("Starting a Read from the Group Store")
	keyA, keyB := murmur3.Sum128([]byte(g))
	childKeyA, childKeyB := murmur3.Sum128([]byte(m))
	_, value, err := fsws.gstore.Read(context.Background(), keyA, keyB, childKeyA, childKeyB, nil)
	if store.IsNotFound(err) {
		log.Printf(" Not Found  Key: %d, %d  ChildKey: %d, %d", keyA, keyB, childKeyA, childKeyB)
		return "", nil
	} else if err != nil {
		return "", err
	}
	log.Println("Successfully read an item from the Group Store")
	return fmt.Sprintf("%s", value), nil
}
Exemple #7
0
// ShowFS ...
func (s *FileSystemAPIServer) ShowFS(ctx context.Context, r *fb.ShowFSRequest) (*fb.ShowFSResponse, error) {
	var status string
	var acctData AcctPayLoad
	var fsData FileSysPayLoad
	var fsDataB []byte
	var err error
	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		fmt.Println(pr.Addr)
	}
	// getAcct data
	acctData, err = s.getAcct("/acct", r.Acctnum)
	if err != nil {
		log.Printf("Error %v on lookup for account %s", err, r.Acctnum)
		return nil, err
	}
	// validate token
	if acctData.Token != r.Token {
		return nil, errf(codes.PermissionDenied, "%s", "Invalid Token")
	}

	pKey := fmt.Sprintf("/acct/%s/fs", r.Acctnum)
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(r.FSid))
	_, fsDataB, err = s.fsws.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		return nil, errf(codes.NotFound, "%v", "File System Not Found")
	}
	if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}
	err = json.Unmarshal(fsDataB, &fsData)
	if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}
	// Get list of Addr
	fsData.Addr, err = s.addrList(fsData.ID)
	if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}
	fsDataB, err = json.Marshal(&fsData)
	if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}

	// Prep things to return
	status = "OK"
	return &fb.ShowFSResponse{Payload: string(fsDataB), Status: status}, nil
}
// Helper methods to get data from value and group store
func (o *StoreComms) ReadValue(ctx context.Context, id []byte) ([]byte, error) {
	// TODO: You might want to make this whole area pass in reusable []byte to
	// lessen gc pressure.
	keyA, keyB := murmur3.Sum128(id)
	_, v, err := o.vstore.Read(ctx, keyA, keyB, nil)
	return v, err
}
func (ktbf *groupKTBloomFilter) mayHave(keyA uint64, keyB uint64, childKeyA uint64, childKeyB uint64, timestamp uint64) bool {
	scratch := ktbf.scratch

	binary.BigEndian.PutUint64(scratch[4:], keyA)
	binary.BigEndian.PutUint64(scratch[12:], keyB)
	binary.BigEndian.PutUint64(scratch[20:], childKeyA)
	binary.BigEndian.PutUint64(scratch[28:], childKeyB)
	binary.BigEndian.PutUint64(scratch[36:], timestamp)

	for i := ktbf.kDiv4; i > 0; i-- {
		binary.BigEndian.PutUint32(scratch, ktbf.salt|i)
		h1, h2 := murmur3.Sum128(scratch)
		bit := uint32(h1>>32) % ktbf.m
		if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 {
			return false
		}
		bit = uint32(h1&0xffffffff) % ktbf.m
		if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 {
			return false
		}
		bit = uint32(h2>>32) % ktbf.m
		if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 {
			return false
		}
		bit = uint32(h2&0xffffffff) % ktbf.m
		if ktbf.bits[bit/8]&(1<<(bit%8)) == 0 {
			return false
		}
	}
	return true
}
func (ktbf *groupKTBloomFilter) add(keyA uint64, keyB uint64, childKeyA uint64, childKeyB uint64, timestamp uint64) {
	// CONSIDER: There are optimization opportunities here as the keys can be
	// considered to already have good bit distribution and using a hashing
	// function to mix-in timestamp, salt, and i instead of redoing the whole
	// hash each time would be good to test and benchmark.
	scratch := ktbf.scratch

	binary.BigEndian.PutUint64(scratch[4:], keyA)
	binary.BigEndian.PutUint64(scratch[12:], keyB)
	binary.BigEndian.PutUint64(scratch[20:], childKeyA)
	binary.BigEndian.PutUint64(scratch[28:], childKeyB)
	binary.BigEndian.PutUint64(scratch[36:], timestamp)

	for i := ktbf.kDiv4; i > 0; i-- {
		binary.BigEndian.PutUint32(scratch, ktbf.salt|i)
		h1, h2 := murmur3.Sum128(scratch)
		bit := uint32(h1>>32) % ktbf.m
		ktbf.bits[bit/8] |= 1 << (bit % 8)
		bit = uint32(h1&0xffffffff) % ktbf.m
		ktbf.bits[bit/8] |= 1 << (bit % 8)
		bit = uint32(h2>>32) % ktbf.m
		ktbf.bits[bit/8] |= 1 << (bit % 8)
		bit = uint32(h2&0xffffffff) % ktbf.m
		ktbf.bits[bit/8] |= 1 << (bit % 8)
	}
}
func (o *StoreComms) ReadGroup(ctx context.Context, key []byte) ([]store.ReadGroupItem, error) {
	keyA, keyB := murmur3.Sum128(key)
	items, err := o.gstore.ReadGroup(ctx, keyA, keyB)
	if err != nil {
		return nil, err
	}
	return items, nil
}
func (o *StoreComms) DeleteValueTS(ctx context.Context, id []byte, tsm int64) error {
	keyA, keyB := murmur3.Sum128(id)
	oldTimestampMicro, err := o.vstore.Delete(ctx, keyA, keyB, tsm)
	if oldTimestampMicro >= tsm {
		return ErrStoreHasNewerValue
	}
	return err
}
Exemple #13
0
// HashBytes will return the hash for the provided byte slice.
func HashBytes(b []byte) []byte {
	h1, h2 := murmur3.Sum128(b)
	return []byte{
		byte(h1 >> 56), byte(h1 >> 48), byte(h1 >> 40), byte(h1 >> 32),
		byte(h1 >> 24), byte(h1 >> 16), byte(h1 >> 8), byte(h1),
		byte(h2 >> 56), byte(h2 >> 48), byte(h2 >> 40), byte(h2 >> 32),
		byte(h2 >> 24), byte(h2 >> 16), byte(h2 >> 8), byte(h2),
	}
}
Exemple #14
0
// validateToken ...
func (s *FileSystemAPIServer) validateToken(t string) (string, error) {
	var tData TokenRef
	var aData AcctPayLoad
	var tDataByte []byte
	var aDataByte []byte
	var err error

	// Read Token
	pKeyA, pKeyB := murmur3.Sum128([]byte("/token"))
	cKeyA, cKeyB := murmur3.Sum128([]byte(t))
	_, tDataByte, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		return "", errors.New("Not Found")
	}
	err = json.Unmarshal(tDataByte, &tData)
	if err != nil {
		log.Printf("TOKEN FAILED %v\n", err)
		return "", err
	}

	// Read Account
	pKeyA, pKeyB = murmur3.Sum128([]byte("/acct"))
	cKeyA, cKeyB = murmur3.Sum128([]byte(tData.AcctID))
	_, aDataByte, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		return "", errors.New("Not Found")
	}
	err = json.Unmarshal(aDataByte, &aData)
	if err != nil {
		log.Printf("TOKEN FAILED %v\n", err)
		return "", err
	}

	if tData.TokenID != aData.Token {
		// Log Failed Operation
		log.Printf("TOKEN FAIL %s\n", t)
		return "", errors.New("Invalid Token")
	}

	// Return Account UUID
	// Log Operation
	log.Printf("TOKEN SUCCESS %s\n", tData.AcctID)
	return tData.AcctID, nil
}
Exemple #15
0
func VSTests() {
	vsconfigs := make([]ValueClientConfig, *clients)
	var wg sync.WaitGroup
	for w := 0; w < *clients; w++ {
		vsconfigs[w].addr = *vsServer
		vsconfigs[w].id = w
		vsconfigs[w].count = perClient
		vsconfigs[w].value = &value
		vsconfigs[w].wg = &wg
		vsconfigs[w].wm = make([]*vp.WriteRequest, perClient)
		vsconfigs[w].rm = make([]*vp.ReadRequest, perClient)
		for k := 0; k < perClient; k++ {
			vsconfigs[w].wm[k] = &vp.WriteRequest{}
			vsconfigs[w].rm[k] = &vp.ReadRequest{}
			vsconfigs[w].wm[k].KeyA, vsconfigs[w].wm[k].KeyB = murmur3.Sum128([]byte(fmt.Sprintf("somethingtestkey%d-%d", vsconfigs[w].id, k)))
			vsconfigs[w].wm[k].TimestampMicro = brimtime.TimeToUnixMicro(time.Now())
			vsconfigs[w].rm[k].KeyA = vsconfigs[w].wm[k].KeyA
			vsconfigs[w].rm[k].KeyB = vsconfigs[w].wm[k].KeyB
		}
	}
	log.Println("ValueStore Key/hash generation complete. Spawning tests.")

	// ValueStore Tests
	if *vsWriteTest {
		t := time.Now()
		for w := 0; w < *clients; w++ {
			wg.Add(1)
			if *streamTest {
				go ValueStreamWrite(&vsconfigs[w])
			} else {
				go ValueWrite(&vsconfigs[w])
			}
		}
		wg.Wait()
		log.Println("Issued", *clients*perClient, "VS WRITES")
		ts := time.Since(t).Seconds()
		log.Println("Total run time was:", ts, "seconds")
		log.Printf("Per second: %.2f\n", float64(*clients*perClient)/ts)
	}
	if *vsReadTest {
		t := time.Now()
		for w := 0; w < *clients; w++ {
			wg.Add(1)
			if *streamTest {
				go ValueStreamRead(&vsconfigs[w])
			} else {
				go ValueRead(&vsconfigs[w])
			}
		}
		wg.Wait()
		log.Println("Issued", *clients*perClient, "VS READS")
		ts := time.Since(t).Seconds()
		log.Println("Total run time was:", ts, "seconds")
		log.Printf("Per second: %.2f\n", float64(*clients*perClient)/ts)
	}
}
Exemple #16
0
// GrantAddrFS ...
func (s *FileSystemAPIServer) GrantAddrFS(ctx context.Context, r *pb.GrantAddrFSRequest) (*pb.GrantAddrFSResponse, error) {
	var err error
	var addrData AddrRef
	var addrByte []byte
	srcAddr := ""

	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}
	// validate token
	_, err = s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s GRANT FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}

	// GRANT an file system entry for the addr
	// 		write /fs/FSID/addr			addr						AddrRef
	pKey := fmt.Sprintf("/fs/%s/addr", r.FSid)
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	addrData.Addr = r.Addr
	addrData.FSID = r.FSid
	addrByte, err = json.Marshal(addrData)
	if err != nil {
		log.Printf("%s GRANT FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, addrByte)
	if err != nil {
		log.Printf("%s GRANT FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}

	// return Addr was Granted
	// Log Operation
	log.Printf("%s GRANT SUCCESS %s %s\n", srcAddr, r.FSid, r.Addr)
	return &pb.GrantAddrFSResponse{Data: r.FSid}, nil
}
func (o *StoreComms) WriteValue(ctx context.Context, id, data []byte) error {
	keyA, keyB := murmur3.Sum128(id)
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	oldTimestampMicro, err := o.vstore.Write(ctx, keyA, keyB, timestampMicro, data)
	if err != nil {
		return err
	}
	if oldTimestampMicro >= timestampMicro {
		return ErrStoreHasNewerValue
	}
	return nil
}
Exemple #18
0
// LookupAddrFS ...
func (s *FileSystemAPIServer) LookupAddrFS(ctx context.Context, r *fb.LookupAddrFSRequest) (*fb.LookupAddrFSResponse, error) {
	var err error
	var status string

	// Check if IP is assigned to the file system
	parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid)
	pKeyA, pKeyB := murmur3.Sum128([]byte(parentKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(r.Addr))

	_, _, err = s.fsws.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		log.Printf("/fs/%s/addr/%s did not exist to delete", r.FSid, r.Addr)
		return nil, errf(codes.NotFound, "%s", "Addr not found")
	} else if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}
	//
	// Verify Account is activea
	status = "OK"
	return &fb.LookupAddrFSResponse{Status: status}, nil
}
Exemple #19
0
// RevokeAddrFS ...
func (s *FileSystemAPIServer) RevokeAddrFS(ctx context.Context, r *fb.RevokeAddrFSRequest) (*fb.RevokeAddrFSResponse, error) {
	var status string
	var err error
	var acctData AcctPayLoad
	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		fmt.Println(pr.Addr)
	}
	// getAcct data
	acctData, err = s.getAcct("/acct", r.Acctnum)
	if err != nil {
		log.Printf("Error %v on lookup for account %s", err, r.Acctnum)
		return nil, errf(codes.NotFound, "%v", err)
	}
	// validate token
	if acctData.Token != r.Token {
		return nil, errf(codes.PermissionDenied, "%s", "Invalid Token")
	}
	parentKey := fmt.Sprintf("/fs/%s/addr", r.FSid)
	childKey := r.Addr

	parentKeyA, parentKeyB := murmur3.Sum128([]byte(parentKey))
	childKeyA, childKeyB := murmur3.Sum128([]byte(childKey))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	// Delete addr
	_, err = s.fsws.gstore.Delete(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro)
	if store.IsNotFound(err) {
		log.Printf("/fs/%s/addr/%s did not exist to delete", r.FSid, r.Addr)
		return nil, errf(codes.NotFound, "%s", "Addr not found")
	} else if err != nil {
		return nil, errf(codes.Internal, "%s", err)
	}
	// DO stuff
	status = fmt.Sprintf("addr %s for filesystem %s with account id %s was revoked", r.Addr, r.FSid, r.Acctnum)
	return &fb.RevokeAddrFSResponse{Status: status}, nil
}
Exemple #20
0
func EtagMatch(w http.ResponseWriter, r *http.Request, contents []byte) bool {
	tHash := time.Now()
	h1, h2 := murmur3.Sum128(contents)
	etag := fmt.Sprintf(`"%x%x"`, h1, h2)
	glog.V(3).Infof("PERF: murmurm3 generation time: %v\n", time.Now().Sub(tHash))

	ifNoneMatch := r.Header.Get("If-None-Match")
	if ifNoneMatch == etag {
		w.WriteHeader(304)
		return true
	}

	w.Header().Set("Etag", etag)
	return false
}
Exemple #21
0
func (self *Hash) Extrude(result []byte) {
	h1, h2 := murmur3.Sum128((*bytes.Buffer)(self).Bytes())
	result[0] = byte(h1 >> 56)
	result[1] = byte(h1 >> 48)
	result[2] = byte(h1 >> 40)
	result[3] = byte(h1 >> 32)
	result[4] = byte(h1 >> 24)
	result[5] = byte(h1 >> 16)
	result[6] = byte(h1 >> 8)
	result[7] = byte(h1)
	result[8] = byte(h2 >> 56)
	result[9] = byte(h2 >> 48)
	result[10] = byte(h2 >> 40)
	result[11] = byte(h2 >> 32)
	result[12] = byte(h2 >> 24)
	result[13] = byte(h2 >> 16)
	result[14] = byte(h2 >> 8)
	result[15] = byte(h2)
}
Exemple #22
0
// grpc Group Store functions
// lookupAccount ...
func (fsws *FileSystemWS) readGroupGStore(g string) (string, error) {
	log.Println("Starting a Group Store Lookup")
	keyA, keyB := murmur3.Sum128([]byte(g))
	items, err := fsws.gstore.ReadGroup(context.Background(), keyA, keyB)
	if store.IsNotFound(err) {
		log.Printf("Not Found: %s", g)
		return "", nil
	} else if err != nil {
		return "", err
	}
	// Build a list of accounts.
	log.Println("Build a list of file systems")
	m := make([]string, len(items))
	for k, v := range items {
		m[k] = fmt.Sprintf("%s", v.Value)
	}
	log.Println("Returning a list of file systems")
	return fmt.Sprintf(strings.Join(m, "|")), nil
}
Exemple #23
0
// addrList ...
func (s *FileSystemAPIServer) addrList(fsid string) ([]string, error) {
	var ap AddrPayLoad
	pKey := fmt.Sprintf("/fs/%s/addr", fsid)
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	items, err := s.fsws.gstore.ReadGroup(context.Background(), pKeyA, pKeyB)
	if store.IsNotFound(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	l := make([]string, len(items))
	for k, v := range items {
		err = json.Unmarshal(v.Value, &ap)
		if err != nil {
			return nil, err
		}
		l[k] = ap.Addr
	}
	return l, nil
}
func (o *StoreComms) ReadGroupItem(ctx context.Context, key, childKey []byte) ([]byte, error) {
	childKeyA, childKeyB := murmur3.Sum128(childKey)
	return o.ReadGroupItemByKey(ctx, key, childKeyA, childKeyB)
}
Exemple #25
0
// CreateFS ...
func (s *FileSystemAPIServer) CreateFS(ctx context.Context, r *pb.CreateFSRequest) (*pb.CreateFSResponse, error) {
	var err error
	var acctID string
	srcAddr := ""
	var fsRef FileSysRef
	var fsRefByte []byte
	var fsSysAttr FileSysAttr
	var fsSysAttrByte []byte

	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}

	// Validate Token
	acctID, err = s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s CREATE FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}

	fsID := uuid.NewV4().String()
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	// Write file system reference entries.
	// write /fs 								FSID						FileSysRef
	pKey := "/fs"
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(fsID))
	fsRef.AcctID = acctID
	fsRef.FSID = fsID
	fsRefByte, err = json.Marshal(fsRef)
	if err != nil {
		log.Printf("%s  CREATE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsRefByte)
	if err != nil {
		log.Printf("%s CREATE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	// write /acct/acctID				FSID						FileSysRef
	pKey = fmt.Sprintf("/acct/%s", acctID)
	pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
	_, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsRefByte)
	if err != nil {
		log.Printf("%s CREATE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	// Write file system attributes
	// write /fs/FSID						name						FileSysAttr
	pKey = fmt.Sprintf("/fs/%s", fsID)
	pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB = murmur3.Sum128([]byte("name"))
	fsSysAttr.Attr = "name"
	fsSysAttr.Value = r.FSName
	fsSysAttr.FSID = fsID
	fsSysAttrByte, err = json.Marshal(fsSysAttr)
	if err != nil {
		log.Printf("%s  CREATE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.gstore.Write(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, timestampMicro, fsSysAttrByte)
	if err != nil {
		log.Printf("%s CREATE FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}

	// Return File System UUID
	// Log Operation
	log.Printf("%s CREATE SUCCESS %s\n", srcAddr, fsID)
	return &pb.CreateFSResponse{Data: fsID}, nil
}
func (o *StoreComms) ReadGroupItemByKey(ctx context.Context, key []byte, childKeyA, childKeyB uint64) ([]byte, error) {
	keyA, keyB := murmur3.Sum128(key)
	_, v, err := o.gstore.Read(ctx, keyA, keyB, childKeyA, childKeyB, nil)
	return v, err
}
Exemple #27
0
// ListFS ...
func (s *FileSystemAPIServer) ListFS(ctx context.Context, r *pb.ListFSRequest) (*pb.ListFSResponse, error) {
	srcAddr := ""
	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}
	// Validate Token
	acctID, err := s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s LIST FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}

	var value []byte
	var fsRef FileSysRef
	var addrData AddrRef
	var fsAttrData FileSysAttr
	var aList []string

	// Read Group /acct/acctID				_						FileSysRef
	pKey := fmt.Sprintf("/acct/%s", acctID)
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	list, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB)
	if err != nil {
		log.Printf("%s LIST FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	fsList := make([]FileSysMeta, len(list))
	for k, v := range list {
		clear(&fsRef)
		clear(&addrData)
		clear(&fsAttrData)
		clear(&aList)
		err = json.Unmarshal(v.Value, &fsRef)
		if err != nil {
			log.Printf("%s LIST FAILED %v\n", srcAddr, err)
			return nil, errf(codes.Internal, "%v", err)
		}
		fsList[k].AcctID = acctID
		fsList[k].ID = fsRef.FSID

		// Get File System Name
		pKey = fmt.Sprintf("/fs/%s", fsList[k].ID)
		pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
		cKeyA, cKeyB := murmur3.Sum128([]byte("name"))
		_, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
		if store.IsNotFound(err) {
			log.Printf("%s LIST FAILED %s NAMENOTFOUND", srcAddr, fsList[k].ID)
			return nil, errf(codes.NotFound, "%v", "File System Name Not Found")
		}
		if err != nil {
			log.Printf("%s LIST FAILED %v\n", srcAddr, err)
			return nil, errf(codes.Internal, "%v", err)
		}
		err = json.Unmarshal(value, &fsAttrData)
		if err != nil {
			log.Printf("%s LIST FAILED %v\n", srcAddr, err)
			return nil, errf(codes.Internal, "%v", err)
		}
		fsList[k].Name = fsAttrData.Value

		// Get List of addrs
		pKey = fmt.Sprintf("/fs/%s/addr", fsList[k].ID)
		pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
		items, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB)
		if !store.IsNotFound(err) {
			// No addr granted
			aList = make([]string, len(items))
			for sk, sv := range items {
				err = json.Unmarshal(sv.Value, &addrData)
				if err != nil {
					log.Printf("%s LIST FAILED %v\n", srcAddr, err)
					return nil, errf(codes.Internal, "%v", err)
				}
				aList[sk] = addrData.Addr
			}
		}
		if err != nil {
			log.Printf("%s LIST FAILED %v\n", srcAddr, err)
			return nil, errf(codes.Internal, "%v", err)
		}
		fsList[k].Addr = aList
	}

	// Return a File System List
	fsListJSON, jerr := json.Marshal(&fsList)
	if jerr != nil {
		return nil, errf(codes.Internal, "%s", jerr)
	}
	// Log Operation
	log.Printf("%s LIST SUCCESS %s\n", srcAddr, acctID)
	return &pb.ListFSResponse{Data: string(fsListJSON)}, nil
}
Exemple #28
0
// ShowFS ...
func (s *FileSystemAPIServer) ShowFS(ctx context.Context, r *pb.ShowFSRequest) (*pb.ShowFSResponse, error) {
	var err error
	var acctID string
	srcAddr := ""

	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		srcAddr = pr.Addr.String()
	}

	// Validate Token
	acctID, err = s.validateToken(r.Token)
	if err != nil {
		log.Printf("%s SHOW FAILED %s\n", srcAddr, "PermissionDenied")
		return nil, errf(codes.PermissionDenied, "%v", "Invalid Token")
	}

	var fs FileSysMeta
	var value []byte
	var fsRef FileSysRef
	var addrData AddrRef
	var fsAttrData FileSysAttr
	var aList []string
	fs.ID = r.FSid

	// Read FileSysRef entry to determine if it exists
	pKey := fmt.Sprintf("/fs")
	pKeyA, pKeyB := murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB := murmur3.Sum128([]byte(fs.ID))
	_, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		log.Printf("%s SHOW FAILED %s NOTFOUND", srcAddr, r.FSid)
		return nil, errf(codes.NotFound, "%v", "Not Found")
	}
	if err != nil {
		log.Printf("%s SHOW FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	err = json.Unmarshal(value, &fsRef)
	if err != nil {
		log.Printf("%s SHOW FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}

	// Validate Token/Account own the file system
	if fsRef.AcctID != acctID {
		log.Printf("$s SHOW FAILED %v ACCOUNT MISMATCH", fs.ID)
		return nil, errf(codes.FailedPrecondition, "%v", "Account Mismatch")
	}
	fs.AcctID = fsRef.AcctID

	// Read the file system attributes
	// group-lookup /fs			FSID
	//		Iterate over all the atributes
	pKey = fmt.Sprintf("/fs/%s", fs.ID)
	pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
	cKeyA, cKeyB = murmur3.Sum128([]byte("name"))
	_, value, err = s.gstore.Read(context.Background(), pKeyA, pKeyB, cKeyA, cKeyB, nil)
	if store.IsNotFound(err) {
		log.Printf("%s SHOW FAILED %s NAMENOTFOUND", srcAddr, r.FSid)
		return nil, errf(codes.NotFound, "%v", "File System Name Not Found")
	}
	if err != nil {
		log.Printf("%s SHOW FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	err = json.Unmarshal(value, &fsAttrData)
	if err != nil {
		log.Printf("%s SHOW FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	fs.Name = fsAttrData.Value

	// Read list of granted ip addresses
	// group-lookup printf("/fs/%s/addr", FSID)
	pKey = fmt.Sprintf("/fs/%s/addr", fs.ID)
	pKeyA, pKeyB = murmur3.Sum128([]byte(pKey))
	items, err := s.gstore.ReadGroup(context.Background(), pKeyA, pKeyB)
	if !store.IsNotFound(err) {
		// No addr granted
		aList = make([]string, len(items))
		for k, v := range items {
			err = json.Unmarshal(v.Value, &addrData)
			if err != nil {
				log.Printf("%s LIST FAILED %v\n", srcAddr, err)
				return nil, errf(codes.Internal, "%v", err)
			}
			aList[k] = addrData.Addr
		}
	}
	if err != nil {
		log.Printf("%s SHOW FAILED %v\n", srcAddr, err)
		return nil, errf(codes.Internal, "%v", err)
	}
	fs.Addr = aList

	// Return File System
	fsJSON, jerr := json.Marshal(&fs)
	if jerr != nil {
		return nil, errf(codes.Internal, "%s", jerr)
	}
	// Log Operation
	log.Printf("%s SHOW SUCCESS %s\n", srcAddr, r.FSid)
	return &pb.ShowFSResponse{Data: string(fsJSON)}, nil
}
Exemple #29
0
// CreateFS ...
func (s *FileSystemAPIServer) CreateFS(ctx context.Context, r *fb.CreateFSRequest) (*fb.CreateFSResponse, error) {
	var status string
	var result string
	var acctData AcctPayLoad
	var newFS FileSysPayLoad
	var acctRef AcctRefPayload
	var acctRefB []byte
	var err error
	var dataB []byte
	// Get incomming ip
	pr, ok := peer.FromContext(ctx)
	if ok {
		fmt.Println(pr.Addr)
	}
	// getAcct data
	acctData, err = s.getAcct("/acct", r.Acctnum)
	if err != nil {
		log.Printf("Error %v on lookup for account %s", err, r.Acctnum)
		return nil, err
	}
	// validate token
	if acctData.Token != r.Token {
		return nil, errf(codes.PermissionDenied, "%s", "Invalid Token")
	}
	// Check for to see if file system name exists
	fs := fmt.Sprintf("/acct/%s/fs", acctData.ID)
	err = s.dupNameCheck(fs, r.FSName)
	if err != nil {
		log.Printf("Precondition Failed: %v\n...", err)
		return nil, errf(codes.FailedPrecondition, "%v", err)
	}
	// File system values
	parentKey := fmt.Sprintf("/acct/%s/fs", r.Acctnum)
	childKey := uuid.NewV4().String()
	newFS.ID = childKey
	newFS.AcctID = r.Acctnum
	newFS.Name = r.FSName
	newFS.SizeInBytes = 107374182400
	newFS.Status = "active"
	newFS.CreateDate = time.Now().Unix()
	newFS.DeleteDate = 0
	//write file system
	dataB, err = json.Marshal(newFS)
	if err != nil {
		log.Printf("Marshal Error: %v\n...", err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.fsws.writeGStore(parentKey, childKey, dataB)
	if err != nil {
		log.Printf("Write Error: %v", err)
		return nil, errf(codes.Internal, "%v", err)
	}
	// Write special filesystem look up entry
	// "/fs"	"[file system uuid]"		{"id": "[filesystem uuid]", "acctid": "[account uuid]"}
	parentKeyA, parentKeyB := murmur3.Sum128([]byte("/fs"))
	childKeyA, childKeyB := murmur3.Sum128([]byte(newFS.ID))
	timestampMicro := brimtime.TimeToUnixMicro(time.Now())
	acctRef.ID = newFS.ID
	acctRef.AcctID = r.Acctnum
	acctRefB, err = json.Marshal(acctRef)
	if err != nil {
		log.Printf("Marshal Error: %v\n...", err)
		return nil, errf(codes.Internal, "%v", err)
	}
	_, err = s.fsws.gstore.Write(context.Background(), parentKeyA, parentKeyB, childKeyA, childKeyB, timestampMicro, acctRefB)
	if err != nil {
		log.Printf("Write Error: %v", err)
		return nil, errf(codes.Internal, "%v", err)
	}
	// Prep reults to return
	status = "OK"
	result = fmt.Sprintf("File System %s was created from Account %s", childKey, r.Acctnum)
	return &fb.CreateFSResponse{Payload: result, Status: status}, nil
}
Exemple #30
0
func (c *Client) parseValueCmd(line string) (string, error) {
	if c.vconn == nil {
		err := c.getValueClient()
		if err != nil {
			return "", err
		}
	}
	split := strings.SplitN(line, " ", 2)
	cmd := split[0]
	if len(split) != 2 {
		if cmd == "exit" {
			return "", fmt.Errorf("Exiting..")
		}
		if cmd == "help" {
			return c.printHelp(), nil
		}
		return c.printHelp(), nil
	}
	args := split[1]
	switch cmd {
	case "write":
		sarg := strings.SplitN(args, " ", 2)
		if len(sarg) < 2 {
			return fmt.Sprintf("write needs key and value: `write somekey some value thing here`"), nil
		}
		keyA, keyB := murmur3.Sum128([]byte(sarg[0]))
		value := []byte(sarg[1])
		timestampMicro := brimtime.TimeToUnixMicro(time.Now())
		oldTimestampMicro, err := c.vstore.Write(context.Background(), keyA, keyB, timestampMicro, value)
		if err != nil {
			return "", err
		}
		return fmt.Sprintf("WRITE TIMESTAMPMICRO: %d\nPREVIOUS TIMESTAMPMICRO: %d", timestampMicro, oldTimestampMicro), nil
	case "read":
		keyA, keyB := murmur3.Sum128([]byte(args))
		timestampMicro, value, err := c.vstore.Read(context.Background(), keyA, keyB, nil)
		if store.IsNotFound(err) {
			return fmt.Sprintf("not found"), nil
		} else if err != nil {
			return "", err
		}
		return fmt.Sprintf("TIMESTAMPMICRO: %d\nVALUE: %s", timestampMicro, value), nil
	case "delete":
		keyA, keyB := murmur3.Sum128([]byte(args))
		timestampMicro := brimtime.TimeToUnixMicro(time.Now())
		oldTimestampMicro, err := c.vstore.Delete(context.Background(), keyA, keyB, timestampMicro)
		if err != nil {
			return "", err
		}
		return fmt.Sprintf("TIMESTAMPMICRO: %d\nOLD TIMESTAMPMICRO: %d", timestampMicro, oldTimestampMicro), nil
	case "lookup":
		keyA, keyB := murmur3.Sum128([]byte(args))
		timestampMicro, length, err := c.vstore.Lookup(context.Background(), keyA, keyB)
		if store.IsNotFound(err) {
			return fmt.Sprintf("not found"), nil
		} else if err != nil {
			return "", err
		}
		return fmt.Sprintf("TIMESTAMPMICRO: %d\nLENGTH: %d", timestampMicro, length), nil
	case "mode":
		if args == "value" {
			return fmt.Sprintf("Already in value store mode"), nil
		}
		if args == "group" {
			c.gmode = true
			return fmt.Sprintf("Switched to group mode"), nil
		}
		return fmt.Sprintf("Valid modes are: value | group"), nil
	case "exit":
		log.Println("exit")
		return "", fmt.Errorf("Exiting..")
	}
	return c.printHelp(), nil
}