func (spec specParentDataset) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "dataset-gateway",
}
e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})
// check for existing link - there can be only be one
re := g.OutWith(src.ID, q.Qbe(system.EType("dataset-gateway")))
if len(re) == 1 {
success = true
e = re[0]
// TODO semantics should preclude this from being able to change, but doing it dirty means force-setting it anyway for now
} else {
// no existing link found; search for proc directly
envid, _, _ := findEnv(g, src)
rv := g.PredecessorsWith(envid, q.Qbv(system.VType("parent-dataset"), "name", spec.Name))
if len(rv) != 0 { // >1 shouldn't be possible
success = true
e.Target = rv[0].ID
}
}
return
}
func (spec specUnixDomainListener) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
// check for existing edge; this one is quite straightforward
re := g.OutWith(src.ID, q.Qbe(system.EType("listening"), "type", "unix", "path", spec.Path))
if len(re) == 1 {
return re[0], true
}
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "listening",
}
e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.Path})
envid, _, hasenv := findEnv(g, src)
if hasenv {
rv := g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "unix", "path", spec.Path))
if len(rv) == 1 {
success = true
e.Target = rv[0].ID
}
}
return
}
func (spec specGitCommitParent) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "parent-commit",
}
re := g.OutWith(src.ID, q.Qbe(system.EType("parent-commit"), "pnum", spec.ParentNum))
if len(re) > 0 {
success = true
e.Target = re[0].Target
e.Props = re[0].Props
// FIXME evidence of a problem here - since we're using pnum as the deduping identifier, there's no
// way it could also sensibly change its MsgSrc value. This is very much a product of the intensional/extensional
// identity problem: what does it mean to have the identifying data change? is it now a new thing? was it the old thing,
// and it underwent a transition into the new thing? or is there no distinction between the old and new thing?
e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
e.ID = re[0].ID
} else {
rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
if len(rv) == 1 {
success = true
e.Target = rv[0].ID
e.Props = e.Props.Set("pnum", system.Property{MsgSrc: mid, Value: spec.ParentNum})
e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
}
}
return
}
func (spec specLocalLogic) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "logic-link",
}
// search for existing link
re := g.OutWith(src.ID, q.Qbe(system.EType("logic-link"), "path", spec.Path))
if len(re) == 1 {
// TODO don't set the path prop again, it's the unique id...meh, same question here w/uniqueness as above
success = true
e = re[0]
return
}
// no existing link found, search for proc directly
envid, _, _ := findEnv(g, src)
rv := g.PredecessorsWith(envid, q.Qbv(system.VType("logic-state"), "path", spec.Path))
if len(rv) == 1 {
success = true
e.Target = rv[0].ID
}
return
}
func (spec specCommit) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "version",
}
e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
re := g.OutWith(src.ID, q.Qbe(system.EType("version")))
if len(re) > 0 {
sha1, _ := re[0].Props.Lookup("sha1")
e.ID = re[0].ID // FIXME setting the id to non-0 AND failing is currently unhandled
if sha1.(system.Property).Value == spec.Sha1 {
success = true
e.Target = re[0].Target
} else {
rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
if len(rv) == 1 {
success = true
e.Target = rv[0].ID
}
}
} else {
rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
if len(rv) == 1 {
success = true
e.Target = rv[0].ID
}
}
return
}
func findMatchingEnvId(g system.CoreGraph, edge system.StdEdge, vtv system.VertexTupleVector) uint64 {
for _, candidate := range vtv {
for _, edge2 := range g.OutWith(candidate.ID, q.Qbe(system.EType("envlink"))) {
if edge2.Target == edge.Target {
return candidate.ID
}
}
}
return 0
}
// Searches the given vertex's out-edges to find its environment's vertex id.
//
// Also conveniently initializes a StandardEdge to the standard zero-state for an envlink.
func findEnv(g system.CoreGraph, vt system.VertexTuple) (vid uint64, edge system.StdEdge, success bool) {
edge = system.StdEdge{
Source: vt.ID,
Props: ps.NewMap(),
EType: "envlink",
}
if vt.ID != 0 {
re := g.OutWith(vt.ID, q.Qbe(system.EType("envlink")))
if len(re) == 1 {
vid, edge, success = re[0].Target, re[0], true
}
}
return
}
func (spec DataAlpha) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
// TODO this makes a loop...are we cool with that?
success = true // impossible to fail here
e = system.StdEdge{
Source: src.ID,
Target: src.ID,
Props: ps.NewMap(),
EType: "data-provenance",
}
re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
if len(re) == 1 {
e = re[0]
}
return
}
func datasetUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
vtv := g.VerticesWith(q.Qbv(system.VType("dataset"), "name", u.Vertex().Properties()["name"]))
if len(vtv) == 0 {
return 0
}
spec := u.ScopingSpecs()[0].(specDatasetHierarchy)
el, success := spec.Environment.Resolve(g, 0, emptyVT(u.Vertex()))
// FIXME scoping edge resolution failure does not mean no match - there could be an orphan
if success {
for _, vt := range vtv {
if id := findMatchingEnvId(g, el, g.SuccessorsWith(vt.ID, q.Qbe(system.EType("dataset-hierarchy")))); id != 0 {
return vt.ID
}
}
}
return 0
}
func (spec DataProvenance) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
// FIXME this presents another weird case where "success" is not binary. We *could*
// find an already-existing data-provenance edge, but then have some net-addr params
// change which cause it to fail to resolve to an environment. If we call that successful,
// then it won't try to resolve again later...though, hm, just call it unsuccessful and
// then try again one more time. Maybe it is fine. THINK IT THROUGH.
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "data-provenance",
}
e.Props = assignAddress(mid, spec.Address, e.Props, false)
re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
if len(re) == 1 {
reresolve := maputil.AnyMatch(e.Props, re[0].Props, "hostname", "ipv4", "ipv6")
e = re[0]
if spec.SnapTime != "" {
e.Props = e.Props.Set("snap-time", system.Property{MsgSrc: mid, Value: spec.SnapTime})
}
if reresolve {
e.Props = assignAddress(mid, spec.Address, e.Props, true)
} else {
return e, true
}
}
envid, found := findEnvironment(g, e.Props)
if !found {
// TODO returning this already-modified edge necessitates that the core system
// disregard 'failed' edges. which should be fine, that should be a guarantee
return e, false
}
e.Target, success = findDataset(g, envid, spec.Dataset)
return
}
func (spec DataLink) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
e = system.StdEdge{
Source: src.ID,
Props: ps.NewMap(),
EType: "datalink",
}
// DataLinks have a 'name' field that is expected to be unique for the source, if present
if spec.Name != "" {
// TODO 'name' is a traditional unique key; a change in it inherently denotes a new edge. how to handle this?
// FIXME this approach just always updates the mid, which is weird?
e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})
re := g.OutWith(src.ID, q.Qbe(system.EType("datalink"), "name", spec.Name))
if len(re) == 1 {
success = true
e = re[0]
}
}
if spec.Type != "" {
e.Props = e.Props.Set("type", system.Property{MsgSrc: mid, Value: spec.Type})
}
if spec.Subset != "" {
e.Props = e.Props.Set("subset", system.Property{MsgSrc: mid, Value: spec.Subset})
}
if spec.Interaction != "" {
e.Props = e.Props.Set("interaction", system.Property{MsgSrc: mid, Value: spec.Interaction})
}
// Special bits: if we have ConnUnix data, eliminate ConnNet data, and vice-versa.
var isLocal bool
if spec.ConnUnix.Path != "" {
isLocal = true
e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.ConnUnix.Path})
e.Props = e.Props.Delete("hostname")
e.Props = e.Props.Delete("ipv4")
e.Props = e.Props.Delete("ipv6")
e.Props = e.Props.Delete("port")
e.Props = e.Props.Delete("proto")
} else {
e.Props = e.Props.Set("port", system.Property{MsgSrc: mid, Value: spec.ConnNet.Port})
e.Props = e.Props.Set("proto", system.Property{MsgSrc: mid, Value: spec.ConnNet.Proto})
// can only be one of hostname, ipv4 or ipv6
if spec.ConnNet.Hostname != "" {
e.Props = e.Props.Set("hostname", system.Property{MsgSrc: mid, Value: spec.ConnNet.Hostname})
} else if spec.ConnNet.Ipv4 != "" {
e.Props = e.Props.Set("ipv4", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv4})
} else {
e.Props = e.Props.Set("ipv6", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv6})
}
}
if success {
return
}
var sock system.VertexTuple
var rv system.VertexTupleVector // just for reuse
// If net, must scan; if local, a bit easier.
if !isLocal {
// First, find the environment vertex
rv = g.VerticesWith(q.Qbv(system.VType("environment")))
var envid uint64
for _, vt := range rv {
if maputil.AnyMatch(e.Props, vt.Vertex.Properties, "hostname", "ipv4", "ipv6") {
envid = vt.ID
break
}
}
// No matching env found, bail out
if envid == 0 {
return
}
// Now, walk the environment's edges to find the vertex representing the port
rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "port", "port", spec.ConnNet.Port).And(q.Qbe(system.EType("envlink"))))
if len(rv) != 1 {
return
}
sock = rv[0]
// With sock in hand, now find its proc
rv = g.PredecessorsWith(sock.ID, q.Qbe(system.EType("listening"), "proto", spec.ConnNet.Proto).And(q.Qbv(system.VType("process"))))
if len(rv) != 1 {
// TODO could/will we ever allow >1?
return
}
} else {
envid, _, exists := findEnv(g, src)
if !exists {
// this is would be a pretty weird case
return
}
// Walk the graph to find the vertex representing the unix socket
rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "path", spec.ConnUnix.Path).And(q.Qbe(system.EType("envlink"))))
if len(rv) != 1 {
return
}
sock = rv[0]
// With sock in hand, now find its proc
rv = g.PredecessorsWith(sock.ID, q.Qbv(system.VType("process")).And(q.Qbe(system.EType("listening"))))
if len(rv) != 1 {
// TODO could/will we ever allow >1?
return
}
}
rv = g.SuccessorsWith(rv[0].ID, q.Qbv(system.VType("parent-dataset")))
// FIXME this absolutely could be more than 1
if len(rv) != 1 {
return
}
dataset := rv[0]
// if the spec indicates a subset, find it
if spec.Subset != "" {
rv = g.PredecessorsWith(rv[0].ID, q.Qbv(system.VType("dataset"), "name", spec.Subset).And(q.Qbe(system.EType("dataset-hierarchy"))))
if len(rv) != 1 {
return
}
dataset = rv[0]
}
// FIXME only recording the final target id is totally broken; see https://github.com/pipeviz/pipeviz/issues/37
// Aaaand we found our target.
success = true
e.Target = dataset.ID
return
}
// Tests adjacentWith(), which effectively tests SuccessorsWith() and PredecessorsWith()
func TestAdjacentWith(t *testing.T) {
g := getGraphFixture()
var result system.VertexTupleVector
// basic, unfiltered tests first to ensure the right data is coming through
// vtx 2 has just one in-edge
result = g.adjacentWith(2, q.Qbv(), true)
if len(result) != 1 {
t.Errorf("Vertex 2 has one predecessor, but got %v vertices", len(result))
}
result = g.PredecessorsWith(2, q.Qbv())
if len(result) != 1 {
t.Errorf("Vertex 2 has one predecessor, but got %v vertices", len(result))
}
// vtx 1 has one out-edge and one in-edge
result = g.adjacentWith(1, q.Qbv(), false)
if len(result) != 1 {
t.Errorf("Vertex 1 has one successor, but got %v vertices", len(result))
}
result = g.SuccessorsWith(1, q.Qbv())
if len(result) != 1 {
t.Errorf("Vertex 1 has one successor, but got %v vertices", len(result))
}
// vtx 5 is an isolate
result = g.adjacentWith(5, q.Qbv(), true)
if len(result) != 0 {
t.Errorf("Vertex 5 has no predecessors, but got %v vertices", len(result))
}
result = g.PredecessorsWith(5, q.Qbv())
if len(result) != 0 {
t.Errorf("Vertex 5 has no predecessors, but got %v vertices", len(result))
}
result = g.adjacentWith(5, q.Qbv(), false)
if len(result) != 0 {
t.Errorf("Vertex 5 has no successors, but got %v vertices", len(result))
}
result = g.SuccessorsWith(5, q.Qbv())
if len(result) != 0 {
t.Errorf("Vertex 5 has no successors, but got %v vertices", len(result))
}
// qbe w/out args should be equivalent
result = g.PredecessorsWith(2, q.Qbe())
if len(result) != 1 {
t.Errorf("Vertex 2 has one predecessor, but got %v vertices (qbe)", len(result))
}
result = g.SuccessorsWith(1, q.Qbe())
if len(result) != 1 {
t.Errorf("Vertex 1 has one successor, but got %v vertices (qbe)", len(result))
}
// deduping: vtx 4 has two in-edges and none out, but those edges are parallel so only one unique vtx
result = g.PredecessorsWith(4, q.Qbv())
if len(result) != 1 {
t.Errorf("Vertex 4 has two in-edges, but only one unique predecessor; however, got %v vertices", len(result))
}
// vtx 3 is on the other side of vtx 4 - three out-edges, but only two uniques
result = g.SuccessorsWith(3, q.Qbv())
if len(result) != 2 {
t.Errorf("Vertex 4 has three out-edges, but only two unique successors; however, got %v vertices", len(result))
}
// filter checks, beginning with edge and/or vertex typing
result = g.SuccessorsWith(3, q.Qbv(system.VType("vt3")))
if len(result) != 1 {
t.Errorf("Vertex 4 has only one unique successor of type \"vt3\"; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.EType("dummy-edge-type2")))
if len(result) != 2 {
t.Errorf("Vertex 4 has two out-edges of \"dummy-edge-type2\" and both point to different vertices, so expecting 2, but got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.EType("dummy-edge-type2")).And(q.Qbv(system.VType("env"))))
if len(result) != 1 {
t.Errorf("Vertex 4 has two unique successors along \"dummy-edge-type2\" out-edges, but only one is vtype \"env\". However, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.EType("dummy-edge-type3")).And(q.Qbv(system.VType("env"))))
if len(result) != 0 {
t.Errorf("Vertex 4 has one unique successor along \"dummy-edge-type3\" out-edges, but it is not an \"env\" type. However, got %v vertices", len(result))
}
// prop-filtering checks
result = g.SuccessorsWith(3, q.Qbv(system.VTypeNone, "prop2", 42))
if len(result) != 2 {
t.Errorf("Vertex 4 has only two unique successors with \"prop2\" at 42; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.ETypeNone, "eprop2", "bar"))
if len(result) != 2 {
t.Errorf("Vertex 4 has two unique successors connected by two out-edges with \"eprop2\" at \"bar\"; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbv(system.VTypeNone, "prop1", "baz", "prop2", 42))
if len(result) != 1 {
t.Errorf("Vertex 4 has only one unique successor with \"prop1\" at \"baz\" and \"prop2\" at 42; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbv(system.VTypeNone, "prop3", "qux", "prop2", 42))
if len(result) != 1 {
t.Errorf("Vertex 4 has only one unique successor with BOTH \"prop3\" at \"qux\" and \"prop2\" at 42; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.ETypeNone, "eprop2", "bar").And(q.Qbv(system.VTypeNone, "prop1", "baz")))
if len(result) != 1 {
t.Errorf("Vertex 4 has only one unique successor with \"prop1\" at \"baz\" along an out-edge with \"eprop2\" at \"bar\"; however, got %v vertices", len(result))
}
result = g.SuccessorsWith(3, q.Qbe(system.ETypeNone, "eprop2", "bar").And(q.Qbv(system.VType("vt3"), "prop1", "baz")))
if len(result) != 1 {
t.Errorf("Vertex 4 has one unique successor of type \"vt3\" with \"prop1\" at \"baz\" along an out-edge with \"eprop2\" at \"bar\"; however, got %v vertices", len(result))
}
}
// Tests arcWith(), which effectively tests both OutWith() and InWith().
func TestOutInArcWith(t *testing.T) {
g := getGraphFixture()
var result system.EdgeVector
// first test zero-case - vtx 5 has no edges
result = g.arcWith(5, q.Qbe(), false)
if len(result) != 0 {
t.Errorf("Vertex 5 has no edges at all, but still got %v out-edge results", len(result))
}
result = g.arcWith(5, q.Qbe(), true)
if len(result) != 0 {
t.Errorf("Vertex 5 has no edges at all, but still got %v in-edge results", len(result))
}
// next test single case - vtx 1 has one in, one out
result = g.arcWith(1, q.Qbe(), true)
if len(result) != 1 {
t.Errorf("Vertex 1 should have one in-edge, but got %v edges", len(result))
}
// ensure InWith behaves same as arcWith + arg
result = g.InWith(1, q.Qbe())
if len(result) != 1 {
t.Errorf("Vertex 1 should have one in-edge, but got %v edges (InWith calls arcWith correctly)", len(result))
}
result = g.arcWith(1, q.Qbe(), false)
if len(result) != 1 {
t.Errorf("Vertex 1 should have one out-edge, but got %v edges", len(result))
}
result = g.OutWith(1, q.Qbe())
if len(result) != 1 {
t.Errorf("Vertex 1 should have one out-edge, but got %v edges (OutWith calls arcWith correctly)", len(result))
}
// last of basic tests - N>1 number of edges
result = g.arcWith(4, q.Qbe(), true)
if len(result) != 2 {
t.Errorf("Vertex 4 has two in-edges, but got %v in-edge results", len(result))
}
result = g.InWith(4, q.Qbe())
if len(result) != 2 {
t.Errorf("Vertex 4 has two in-edges, but got %v in-edge results (InWith calls arcWith correctly)", len(result))
}
result = g.arcWith(3, q.Qbe(), false)
if len(result) != 3 {
t.Errorf("Vertex 3 should have three out-edges, but got %v edges", len(result))
}
result = g.OutWith(3, q.Qbe())
if len(result) != 3 {
t.Errorf("Vertex 3 should have three out-edges, but got %v edges (OutWith calls arcWith correctly)", len(result))
}
result = g.InWith(3, q.Qbe())
if len(result) != 0 {
t.Errorf("Vertex 3 has out-edges but no in-edge; still got %v in-edge results", len(result))
}
// now, tests that actually exercise the filter
result = g.OutWith(3, q.Qbe(system.ETypeNone))
if len(result) != 3 {
t.Errorf("ETypeNone does not correctly matches all edge types - should've gotten 3 out-edges, but got %v edges", len(result))
}
// basic edge type filtering
result = g.OutWith(3, q.Qbe(system.EType("dummy-edge-type2")))
if len(result) != 2 {
t.Errorf("Vertex 2 should have two \"dummy-edge-type2\"-typed out-edges, but got %v edges", len(result))
}
// nonexistent type means no results
result = g.InWith(2, q.Qbe(system.EType("nonexistent-type")))
if len(result) != 0 {
t.Errorf("Vertex 2 should have no edges of a nonexistent type, but got %v edges", len(result))
}
// existing edge type, but not one this vt has
result = g.InWith(3, q.Qbe(system.EType("dummy-edge-type1")))
if len(result) != 0 {
t.Errorf("Vertex 3 has none of the \"dummy-edge-type1\" edges (though it is a real type in the graph); however, got %v edges", len(result))
}
// test prop-checking
result = g.OutWith(3, q.Qbe(system.ETypeNone, "eprop2", "baz"))
if len(result) != 1 {
t.Errorf("Vertex 3 should have one out-edge with \"eprop2\" at \"baz\", but got %v edges", len(result))
}
result = g.OutWith(3, q.Qbe(system.ETypeNone, "eprop2", "bar"))
if len(result) != 2 {
t.Errorf("Vertex 3 should have two out-edges with \"eprop2\" at \"bar\", but got %v edges", len(result))
}
// test multi-prop checking - ensure they\"re ANDed
result = g.OutWith(3, q.Qbe(system.ETypeNone, "eprop2", "bar", "eprop3", 42))
if len(result) != 1 {
t.Errorf("Vertex 3 should have one out-edge with \"eprop2\" at \"bar\" AND \"eprop3\" at 42, but got %v edges", len(result))
}
result = g.OutWith(3, q.Qbe(system.ETypeNone, "eprop2", "baz", "eprop3", 42))
if len(result) != 0 {
// OR would\"ve here would produce 2 edges
t.Errorf("Vertex 3 should have no out-edges with \"eprop2\" at \"baz\" AND \"eprop3\" at 42 , but got %v edges", len(result))
}
result = g.OutWith(3, q.Qbe(system.EType("dummy-edge-type2"), "eprop2", "bar"))
if len(result) != 1 {
t.Errorf("Vertex 3 should have one out-edges that is dummy type2 AND has \"eprop2\" at \"bar\", but got %v edges", len(result))
}
}