func (suite *GraphTestSuite) TestSave(t *C) {
mTime := time.Now()
ni := graph.NodeInfo{
Name: "child",
ParentId: graph.RootNodeId,
Mode: os.FileMode(0755),
MTime: mTime,
Type: "application/json",
}
node, err := suite.ng.NewNodeWithNodeInfo(ni)
t.Check(err, IsNil)
assertProperty := func(expected string, actual cayley.Iterator) {
t.Check(cayley.RawNext(actual), Equals, true)
t.Check(suite.ng.NameOf(actual.Result()), Equals, expected)
}
it := cayley.StartPath(suite.ng, node.Id).Out("isNamed").BuildIterator()
assertProperty("child", it)
it = cayley.StartPath(suite.ng, node.Id).Out("hasMTime").BuildIterator()
assertProperty(mTime.Format(time.RFC3339Nano), it)
it = cayley.StartPath(suite.ng, node.Id).Out("hasMode").BuildIterator()
assertProperty(fmt.Sprint(0755), it)
it = cayley.StartPath(suite.ng, node.Id).Out("hasType").BuildIterator()
assertProperty("application/json", it)
}
// toLayers converts a path leading to one or multiple layers to Layer structs,
// selecting the specified fields
func toLayers(path *path.Path, selectedFields []string) ([]*Layer, error) {
var layers []*Layer
saveFields(path, selectedFields, []string{FieldLayerSuccessors, FieldLayerPackages, FieldLayerInstalledPackages, FieldLayerRemovedPackages})
it, _ := path.BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
layer := Layer{Node: store.NameOf(it.Result())}
for _, selectedField := range selectedFields {
switch selectedField {
case FieldLayerID:
layer.ID = store.NameOf(tags[FieldLayerID])
case FieldLayerParent:
layer.ParentNode = store.NameOf(tags[FieldLayerParent])
case FieldLayerSuccessors:
var err error
layer.SuccessorsNodes, err = toValues(cayley.StartPath(store, layer.Node).In(FieldLayerParent))
if err != nil {
log.Errorf("could not get successors of layer %s: %s.", layer.Node, err.Error())
return nil, err
}
case FieldLayerOS:
layer.OS = store.NameOf(tags[FieldLayerOS])
case FieldLayerPackages:
var err error
it, _ := cayley.StartPath(store, layer.Node).OutWithTags([]string{"predicate"}, FieldLayerInstalledPackages, FieldLayerRemovedPackages).BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
predicate := store.NameOf(tags["predicate"])
if predicate == FieldLayerInstalledPackages {
layer.InstalledPackagesNodes = append(layer.InstalledPackagesNodes, store.NameOf(it.Result()))
} else if predicate == FieldLayerRemovedPackages {
layer.RemovedPackagesNodes = append(layer.RemovedPackagesNodes, store.NameOf(it.Result()))
}
}
if it.Err() != nil {
log.Errorf("could not get installed/removed packages of layer %s: %s.", layer.Node, it.Err())
return nil, err
}
case FieldLayerEngineVersion:
layer.EngineVersion, _ = strconv.Atoi(store.NameOf(tags[FieldLayerEngineVersion]))
default:
panic("unknown selectedField")
}
}
layers = append(layers, &layer)
}
if it.Err() != nil {
log.Errorf("failed query in toLayers: %s", it.Err())
return []*Layer{}, ErrBackendException
}
return layers, nil
}
func (ng *NodeGraph) NodeWithName(parentId, name string) *Node {
namePath := cayley.StartPath(ng, name).In(nameLink)
parentpath := cayley.StartPath(ng, parentId).In(parentLink)
it := namePath.And(parentpath).BuildIterator()
if cayley.RawNext(it) {
return ng.NodeWithId(ng.NameOf(it.Result()))
}
return nil
}
// FindAllLayersByAddedPackageNodes finds and returns all layers that add the
// given packages (by their nodes), selecting the specified fields
func FindAllLayersByAddedPackageNodes(nodes []string, selectedFields []string) ([]*Layer, error) {
layers, err := toLayers(cayley.StartPath(store, nodes...).In(FieldLayerInstalledPackages), selectedFields)
if err != nil {
return []*Layer{}, err
}
return layers, nil
}
// Unlock unlocks a lock specified by its name if I own it
func Unlock(name, owner string) {
unlocked := 0
it, _ := cayley.StartPath(store, name).Has(fieldLockLocked, fieldLockLockedValue).Has(fieldLockLockedBy, owner).Save(fieldLockLockedUntil, fieldLockLockedUntil).BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
t := cayley.NewTransaction()
t.RemoveQuad(cayley.Triple(name, fieldLockLocked, fieldLockLockedValue))
t.RemoveQuad(cayley.Triple(name, fieldLockLockedUntil, store.NameOf(tags[fieldLockLockedUntil])))
t.RemoveQuad(cayley.Triple(name, fieldLockLockedBy, owner))
err := store.ApplyTransaction(t)
if err != nil {
log.Errorf("failed transaction (Unlock): %s", err)
}
unlocked++
}
if it.Err() != nil {
log.Errorf("failed query in Unlock: %s", it.Err())
}
if unlocked > 1 {
// We should never see this, it would mean that our database doesn't ensure quad uniqueness
// and that the entire lock system is jeopardized.
log.Errorf("found inconsistency in Unlock: matched %d times a locked named: %s", unlocked, name)
}
}
// FindAllVulnerabilitiesByFixedIn finds and returns all vulnerabilities that are fixed in the given packages (speficied by their nodes), selecting the specified fields
func FindAllVulnerabilitiesByFixedIn(nodes []string, selectedFields []string) ([]*Vulnerability, error) {
if len(nodes) == 0 {
log.Warning("Could not FindAllVulnerabilitiesByFixedIn with an empty nodes array.")
return []*Vulnerability{}, nil
}
return toVulnerabilities(cayley.StartPath(store, nodes...).In(FieldVulnerabilityFixedIn), selectedFields)
}
func main() {
store, err := cayley.NewGraph("bolt", dbPath, nil)
if err != nil {
fmt.Println("error in creating database", err)
}
path := cayley.StartPath(store, "Article").
In().
Tag("link").
Save("has_image", "image").
Save("has_title", "title").
Save("has_description", "description")
it := path.BuildIterator()
it, _ = it.Optimize()
for graph.Next(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
fmt.Println(store.NameOf(tags["image"]))
fmt.Println(store.NameOf(tags["title"]))
fmt.Println(store.NameOf(tags["description"]))
fmt.Println(store.NameOf(tags["link"]))
}
}
// FindAllPackagesByNodes finds and returns all packages given by their nodes, selecting the specified fields
func FindAllPackagesByNodes(nodes []string, selectedFields []string) ([]*Package, error) {
if len(nodes) == 0 {
return []*Package{}, nil
}
return toPackages(cayley.StartPath(store, nodes...).Has(FieldIs, FieldPackageIsValue), selectedFields)
}
/*
* Searches for articles in the database
* @param tags string[] The tags entered by the user.Currently only entity search has been implemented.
Algorithm in Readme
*/
func SearchForArticles(tags []string, store *cayley.Handle) PairList {
links := make(map[string]int)
for _, tag := range tags {
entities := extractArticleData.GetEntityInfo(tag, "")
if len(entities) == 0 {
continue
}
entitityInfo := entities[0] // Get the First Entity //TODO:- Change to Best Label Match
for _, category := range entitityInfo.Categories {
path := cayley.StartPath(store, category).
In("has_category").
In("has_entity")
it := path.BuildIterator()
it, _ = it.Optimize()
for cayley.RawNext(it) {
link := store.NameOf(it.Result())
links[link] += 10
}
}
}
return sortMapByValue(links)
}
func (s *localSignallingLayer) AddServiceHost(service string, version string, uri string) (common.ServiceHost, error) {
var h, err = s.base.AddServiceHost(service, version, uri)
if err == nil {
var svcDef, finderr = s.base.GetServiceDefinition(service)
if finderr == nil {
// find all affected api versions
var startingNodeID = utils.CreateServiceVersionKey(service, version)
var path = cayley.StartPath(s.graph, startingNodeID).Out(utils.InApiRel)
it := path.BuildIterator()
for cayley.RawNext(it) {
var apiNodeID = s.graph.NameOf(it.Result())
var apiVersion = strings.TrimPrefix(apiNodeID, utils.CreateAPIVersionKey(""))
var event = common.Event{}
event.ActionCode = "ADD_ServiceHost"
event.Data = map[string]string{
"version": apiVersion,
"prefix": svcDef.Prefix,
"uri": uri}
s.signalOutput <- event
}
}
}
return h, err
}
// pruneLocks removes every expired locks from the database
func pruneLocks() {
now := time.Now()
// Delete every expired locks
it, _ := cayley.StartPath(store, "locked").In("locked").Save(fieldLockLockedUntil, fieldLockLockedUntil).Save(fieldLockLockedBy, fieldLockLockedBy).BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
n := store.NameOf(it.Result())
t := store.NameOf(tags[fieldLockLockedUntil])
o := store.NameOf(tags[fieldLockLockedBy])
tt, _ := strconv.ParseInt(t, 10, 64)
if now.Unix() > tt {
log.Debugf("lock %s owned by %s has expired.", n, o)
tr := cayley.NewTransaction()
tr.RemoveQuad(cayley.Triple(n, fieldLockLocked, fieldLockLockedValue))
tr.RemoveQuad(cayley.Triple(n, fieldLockLockedUntil, t))
tr.RemoveQuad(cayley.Triple(n, fieldLockLockedBy, o))
err := store.ApplyTransaction(tr)
if err != nil {
log.Errorf("failed transaction (pruneLocks): %s", err)
continue
}
log.Debugf("lock %s has been successfully pruned.", n)
}
}
if it.Err() != nil {
log.Errorf("failed query in Unlock: %s", it.Err())
}
}
// pruneLocks removes every expired locks from the database
func pruneLocks() {
now := time.Now()
// Delete every expired locks
tr := cayley.NewTransaction()
it, _ := cayley.StartPath(store, "locked").In("locked").Save("locked_until", "locked_until").Save("locked_by", "locked_by").BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
n := store.NameOf(it.Result())
t := store.NameOf(tags["locked_until"])
o := store.NameOf(tags["locked_by"])
tt, _ := strconv.ParseInt(t, 10, 64)
if now.Unix() > tt {
log.Debugf("Lock %s owned by %s has expired.", n, o)
tr.RemoveQuad(cayley.Quad(n, "locked", "locked", ""))
tr.RemoveQuad(cayley.Quad(n, "locked_until", t, ""))
tr.RemoveQuad(cayley.Quad(n, "locked_by", o, ""))
}
}
store.ApplyTransaction(tr)
}
// Lock tries to set a temporary lock in the database.
// If a lock already exists with the given name/owner, then the lock is renewed
//
// Lock does not block, instead, it returns true and its expiration time
// is the lock has been successfully acquired or false otherwise
func Lock(name string, duration time.Duration, owner string) (bool, time.Time) {
pruneLocks()
until := time.Now().Add(duration)
untilString := strconv.FormatInt(until.Unix(), 10)
// Try to get the expiration time of a lock with the same name/owner
currentExpiration, err := toValue(cayley.StartPath(store, name).Has("locked_by", owner).Out("locked_until"))
if err == nil && currentExpiration != "" {
// Renew our lock
if currentExpiration == untilString {
return true, until
}
t := cayley.NewTransaction()
t.RemoveQuad(cayley.Quad(name, "locked_until", currentExpiration, ""))
t.AddQuad(cayley.Quad(name, "locked_until", untilString, ""))
// It is not necessary to verify if the lock is ours again in the transaction
// because if someone took it, the lock's current expiration probably changed and the transaction will fail
return store.ApplyTransaction(t) == nil, until
}
t := cayley.NewTransaction()
t.AddQuad(cayley.Quad(name, "locked", "locked", "")) // Necessary to make the transaction fails if the lock already exists (and has not been pruned)
t.AddQuad(cayley.Quad(name, "locked_until", untilString, ""))
t.AddQuad(cayley.Quad(name, "locked_by", owner, ""))
glog.SetStderrThreshold("FATAL")
success := store.ApplyTransaction(t) == nil
glog.SetStderrThreshold("ERROR")
return success, until
}
// FindAllPackagesByNodes finds and returns all packages given by their nodes, selecting the specified fields
func FindAllPackagesByNodes(nodes []string, selectedFields []string) ([]*Package, error) {
if len(nodes) == 0 {
log.Warning("could not FindAllPackagesByNodes with an empty nodes array.")
return []*Package{}, nil
}
return toPackages(cayley.StartPath(store, nodes...).Has(FieldIs, FieldPackageIsValue), selectedFields)
}
func (nd *Node) graphValue(key string) (value string) {
it := cayley.StartPath(nd.graph, nd.Id).Out(key).BuildIterator()
if cayley.RawNext(it) {
value = nd.graph.NameOf(it.Result())
}
return
}
func (s *SocketServer) NotifyStop(t int64) {
stopAt := strconv.FormatInt(t, 10)
p := cayley.StartPath(s.storage, stopAt).In("free at")
it := p.BuildIterator()
for cayley.RawNext(it) {
s.BroadcastTo(RoomName, "stop", s.storage.NameOf(it.Result()))
}
}
func (s *SocketServer) NotifyEnable(t int64) {
stopedAt := strconv.FormatInt(t-models.Wait5Minutes, 10)
p := cayley.StartPath(s.storage, stopedAt).In("free at")
it := p.BuildIterator()
for cayley.RawNext(it) {
//TODO send to only one user
s.BroadcastTo(RoomName, "enable", s.storage.NameOf(it.Result()))
}
}
func (a *ACL) GetProfilePerm(sourceprofileid, targetprofile string) []string {
p := cayley.StartPath(a.profilestore, sourceprofileid).Out(targetprofile)
it := p.BuildIterator()
r := []string{}
for cayley.RawNext(it) {
r = append(r, a.profilestore.NameOf(it.Result()))
}
return r
}
func TestToValue(t *testing.T) {
Open(&config.DatabaseConfig{Type: "memstore"})
defer Close()
// toValue()
v, err := toValue(cayley.StartPath(store, "tests").Out("are"))
assert.Nil(t, err, "toValue should work even if the requested path leads to nothing")
assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing")
store.AddQuad(cayley.Triple("tests", "are", "awesome"))
v, err = toValue(cayley.StartPath(store, "tests").Out("are"))
assert.Nil(t, err, "toValue should have worked")
assert.Equal(t, "awesome", v, "toValue did not return the expected value")
store.AddQuad(cayley.Triple("tests", "are", "running"))
v, err = toValue(cayley.StartPath(store, "tests").Out("are"))
assert.NotNil(t, err, "toValue should return an error and an empty string if the path leads to multiple values")
assert.Equal(t, "", v, "toValue should return an error and an empty string if the path leads to multiple values")
// toValues()
vs, err := toValues(cayley.StartPath(store, "CoreOS").Out(fieldIs))
assert.Nil(t, err, "toValues should work even if the requested path leads to nothing")
assert.Len(t, vs, 0, "toValue should return an empty array if the requested path leads to nothing")
words := []string{"powerful", "lightweight"}
for i, word := range words {
store.AddQuad(cayley.Triple("CoreOS", fieldIs, word))
v, err := toValues(cayley.StartPath(store, "CoreOS").Out(fieldIs))
assert.Nil(t, err, "toValues should have worked")
assert.Len(t, v, i+1, "toValues did not return the right amount of values")
for _, e := range words[:i+1] {
assert.Contains(t, v, e, "toValues did not return the values we expected")
}
}
// toValue(s)() and empty values
store.AddQuad(cayley.Triple("bob", "likes", ""))
v, err = toValue(cayley.StartPath(store, "bob").Out("likes"))
assert.Nil(t, err, "toValue should work even if the requested path leads to nothing")
assert.Equal(t, "", v, "toValue should return an empty string if the requested path leads to nothing")
store.AddQuad(cayley.Triple("bob", "likes", "running"))
v, err = toValue(cayley.StartPath(store, "bob").Out("likes"))
assert.NotNil(t, err, "toValue should return an error and an empty string if the path leads to multiple values")
assert.Equal(t, "", v, "toValue should return an error and an empty string if the path leads to multiple values")
store.AddQuad(cayley.Triple("bob", "likes", "swimming"))
va, err := toValues(cayley.StartPath(store, "bob").Out("likes"))
assert.Nil(t, err, "toValues should have worked")
if assert.Len(t, va, 3, "toValues should have returned 2 values") {
assert.Contains(t, va, "running")
assert.Contains(t, va, "swimming")
assert.Contains(t, va, "")
}
}
func (s *Storage) GetUsersFreeAt(t int64) []*User {
freeAt := strconv.FormatInt(t, 10)
users := []*User{}
p := cayley.StartPath(s, freeAt).In("free at")
it := p.BuildIterator()
for cayley.RawNext(it) {
users = append(users, NewUser(s.NameOf(it.Result())))
}
return users
}
func (nd *Node) BlockWithOffset(offset int64) string {
if nd.IsDir() {
return ""
}
it := cayley.StartPath(nd.graph, nd.Id).Out(fmt.Sprint("offset-", offset)).BuildIterator()
if cayley.RawNext(it) {
return nd.graph.NameOf(it.Result())
} else {
return ""
}
}
func (s *Storage) SaveUser(u *User) {
s.AddQuad(cayley.Quad(u.Name, "is", "user", ""))
p := cayley.StartPath(u.getStorage(), u.Name).Out("free at")
iterationTime := u.IterationTime()
it := p.BuildIterator()
for cayley.RawNext(it) {
s.RemoveQuad(cayley.Quad(u.Name, "free at", s.NameOf(it.Result()), ""))
}
s.AddQuad(cayley.Quad(u.Name, "free at", strconv.FormatInt(iterationTime, 10), ""))
}
func (u *User) IterationTime() int64 {
if u.iterationTime == 0 {
p := cayley.StartPath(u.getStorage(), u.Name).Out("free at")
it := p.BuildIterator()
if cayley.RawNext(it) {
u.iterationTime, _ = strconv.ParseInt(u.getStorage().NameOf(it.Result()), 10, 64)
} else {
u.iterationTime = time.Now().Unix() - Wait5Minutes
}
}
return u.iterationTime
}
func (suite *GraphTestSuite) TestWriteData_removesExistingFingerprintForOffset(t *C) {
child, _ := makeNode("child", suite.ng.RootNode.Id, time.Now(), suite.ng)
dat := testutils.RandDat(1024)
t.Check(child.WriteData(dat, 0), IsNil)
dat = testutils.RandDat(1024)
fingerprint := graph.Hash(dat)
t.Check(child.WriteData(dat, 0), IsNil)
it := cayley.StartPath(suite.ng, child.Id).Out("offset-0").BuildIterator()
t.Check(cayley.RawNext(it), Equals, true)
t.Check(suite.ng.NameOf(it.Result()), Equals, fingerprint)
}
// CountNotificationsToSend returns the number of pending notifications
// Note that it also count the locked notifications.
func CountNotificationsToSend() (int, error) {
c := 0
it, _ := cayley.StartPath(store, "notification").In(FieldIs).Has("isSent", strconv.FormatBool(false)).BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
c = c + 1
}
if it.Err() != nil {
log.Errorf("failed query in CountNotificationsToSend: %s", it.Err())
return 0, ErrBackendException
}
return c, nil
}
func (p *mgoPersistenceProvider) loadVersionsByAPI(apiVersion string, graph *cayley.Handle, session *mgo.Session, target common.IServiceRegistry) error {
var startingNodeID = utils.CreateAPIVersionKey(apiVersion)
var path = cayley.StartPath(graph, startingNodeID).Out(utils.ContainsVersionRel)
it := path.BuildIterator()
for cayley.RawNext(it) {
var version serviceVersion
err := session.DB("test").C("serviceVersions").Find(bson.M{"_id": graph.NameOf(it.Result())}).One(&version)
if err != nil {
return err
}
target.LinkServiceToAPI(version.Service, version.Version, apiVersion)
}
return nil
}
// FindAllVulnerabilitiesByFixedIn finds and returns all vulnerabilities that are fixed in the given packages (speficied by their nodes), selecting the specified fields
func FindAllVulnerabilitiesByFixedIn(nodes []string, selectedFields []string) ([]*Vulnerability, error) {
if len(nodes) == 0 {
log.Warning("Could not FindAllVulnerabilitiesByFixedIn with an empty nodes array.")
return []*Vulnerability{}, nil
}
// Construct path, potentially saving FieldVulnerabilityCausedByPackage
path := cayley.StartPath(store, nodes...)
if utils.Contains(FieldVulnerabilityCausedByPackage, selectedFields) {
path = path.Save(FieldPackageName, FieldVulnerabilityCausedByPackage)
}
path = path.In(FieldVulnerabilityFixedIn)
return toVulnerabilities(path, selectedFields)
}
// FindOneLayerByNode finds and returns a single package by its node, selecting the specified fields
func FindOneLayerByNode(node string, selectedFields []string) (*Layer, error) {
l, err := toLayers(cayley.StartPath(store, node).Has(FieldIs, FieldLayerIsValue), selectedFields)
if err != nil {
return nil, err
}
if len(l) == 1 {
return l[0], nil
}
if len(l) > 1 {
log.Errorf("found multiple layers with identical node [Node: %s]", node)
return nil, ErrInconsistent
}
return nil, cerrors.ErrNotFound
}
// FindOneVulnerability finds and returns a single vulnerability having the given ID selecting the specified fields
func FindOneVulnerability(id string, selectedFields []string) (*Vulnerability, error) {
t := &Vulnerability{ID: id}
v, err := toVulnerabilities(cayley.StartPath(store, t.GetNode()).Has(fieldIs, fieldVulnerabilityIsValue), selectedFields)
if err != nil {
return nil, err
}
if len(v) == 1 {
return v[0], nil
}
if len(v) > 1 {
log.Errorf("found multiple vulnerabilities with identical ID [ID: %s]", id)
return nil, ErrInconsistent
}
return nil, cerrors.ErrNotFound
}
// FindOnePackage finds and returns a single package having the given OS, name and version, selecting the specified fields
func FindOnePackage(OS, name string, version types.Version, selectedFields []string) (*Package, error) {
packageParameter := Package{OS: OS, Name: name, Version: version}
p, err := toPackages(cayley.StartPath(store, packageParameter.GetNode()).Has(FieldIs, FieldPackageIsValue), selectedFields)
if err != nil {
return nil, err
}
if len(p) == 1 {
return p[0], nil
}
if len(p) > 1 {
log.Errorf("found multiple packages with identical data [OS: %s, Name: %s, Version: %s]", OS, name, version)
return nil, ErrInconsistent
}
return nil, cerrors.ErrNotFound
}
