// TODO it would be better to not have to have this here, at all. func emptyVT(v system.ProtoVertex) system.VertexTuple { var props []system.PropPair for k, v := range v.Properties() { props = append(props, pp(k, v)) } return system.VertexTuple{ Vertex: system.NewVertex(v.Type(), 0, props...), InEdges: ps.NewMap(), OutEdges: ps.NewMap(), } }
func (spec specGitCommitParent) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "parent-commit", } re := g.OutWith(src.ID, q.Qbe(system.EType("parent-commit"), "pnum", spec.ParentNum)) if len(re) > 0 { success = true e.Target = re[0].Target e.Props = re[0].Props // FIXME evidence of a problem here - since we're using pnum as the deduping identifier, there's no // way it could also sensibly change its MsgSrc value. This is very much a product of the intensional/extensional // identity problem: what does it mean to have the identifying data change? is it now a new thing? was it the old thing, // and it underwent a transition into the new thing? or is there no distinction between the old and new thing? e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1}) e.ID = re[0].ID } else { rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1)) if len(rv) == 1 { success = true e.Target = rv[0].ID e.Props = e.Props.Set("pnum", system.Property{MsgSrc: mid, Value: spec.ParentNum}) e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1}) } } return }
func (spec specLocalLogic) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "logic-link", } // search for existing link re := g.OutWith(src.ID, q.Qbe(system.EType("logic-link"), "path", spec.Path)) if len(re) == 1 { // TODO don't set the path prop again, it's the unique id...meh, same question here w/uniqueness as above success = true e = re[0] return } // no existing link found, search for proc directly envid, _, _ := findEnv(g, src) rv := g.PredecessorsWith(envid, q.Qbv(system.VType("logic-state"), "path", spec.Path)) if len(rv) == 1 { success = true e.Target = rv[0].ID } return }
func (spec specParentDataset) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "dataset-gateway", } e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name}) // check for existing link - there can be only be one re := g.OutWith(src.ID, q.Qbe(system.EType("dataset-gateway"))) if len(re) == 1 { success = true e = re[0] // TODO semantics should preclude this from being able to change, but doing it dirty means force-setting it anyway for now } else { // no existing link found; search for proc directly envid, _, _ := findEnv(g, src) rv := g.PredecessorsWith(envid, q.Qbv(system.VType("parent-dataset"), "name", spec.Name)) if len(rv) != 0 { // >1 shouldn't be possible success = true e.Target = rv[0].ID } } return }
func (spec specUnixDomainListener) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { // check for existing edge; this one is quite straightforward re := g.OutWith(src.ID, q.Qbe(system.EType("listening"), "type", "unix", "path", spec.Path)) if len(re) == 1 { return re[0], true } e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "listening", } e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.Path}) envid, _, hasenv := findEnv(g, src) if hasenv { rv := g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "unix", "path", spec.Path)) if len(rv) == 1 { success = true e.Target = rv[0].ID } } return }
func (spec specCommit) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "version", } e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1}) re := g.OutWith(src.ID, q.Qbe(system.EType("version"))) if len(re) > 0 { sha1, _ := re[0].Props.Lookup("sha1") e.ID = re[0].ID // FIXME setting the id to non-0 AND failing is currently unhandled if sha1.(system.Property).Value == spec.Sha1 { success = true e.Target = re[0].Target } else { rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1)) if len(rv) == 1 { success = true e.Target = rv[0].ID } } } else { rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1)) if len(rv) == 1 { success = true e.Target = rv[0].ID } } return }
func BenchmarkMergeMessageOneAndTwo(b *testing.B) { var g system.CoreGraph = &coreGraph{vtuples: ps.NewMap()} for i := 0; i < b.N; i++ { g.Merge(0, msgs[0].UnificationForm()) g.Merge(0, msgs[1].UnificationForm()) } }
// Transform a slice of kv pairs into a map. func mapPairs(pairs ...p) (m ps.Map) { m = ps.NewMap() for _, kv := range pairs { m = m.Set(kv.K, kv.V) } return }
// utility func to create a vtTuple. puts edges in the right place by // checking source/target ids. panics if they don't line up! func mkTuple(vid uint64, vtx system.StdVertex, edges ...system.StdEdge) system.VertexTuple { vt := system.VertexTuple{ ID: vid, Vertex: vtx, InEdges: ps.NewMap(), OutEdges: ps.NewMap(), } for _, e := range edges { if e.Source == vid { vt.OutEdges = vt.OutEdges.Set(strconv.FormatUint(e.ID, 10), e) } else if e.Target == vid { vt.InEdges = vt.InEdges.Set(strconv.FormatUint(e.ID, 10), e) } else { panic("edge had neither source nor target of vid") } } return vt }
func fillMapIgnoreZero(msgid uint64, p ...PropPair) ps.Map { m := ps.NewMap() var zero bool var err error for _, pair := range p { if zero, err = isZero(pair.V); !zero && err == nil { m = m.Set(pair.K, Property{MsgSrc: msgid, Value: pair.V}) } } return m }
func TestClone(t *testing.T) { g := &coreGraph{vtuples: ps.NewMap(), vserial: 0} g.vserial = 2 g.vtuples = g.vtuples.Set("foo", "val") var g2 *coreGraph = g.clone() g2.vserial = 4 g2.vtuples = g2.vtuples.Set("foo", "newval") if g.vserial != 2 { t.Errorf("changes in cloned graph propagated back to original") } if val, _ := g2.vtuples.Lookup("foo"); val != "newval" { t.Errorf("map somehow propagated changes back up to original map") } }
// Searches the given vertex's out-edges to find its environment's vertex id. // // Also conveniently initializes a StandardEdge to the standard zero-state for an envlink. func findEnv(g system.CoreGraph, vt system.VertexTuple) (vid uint64, edge system.StdEdge, success bool) { edge = system.StdEdge{ Source: vt.ID, Props: ps.NewMap(), EType: "envlink", } if vt.ID != 0 { re := g.OutWith(vt.ID, q.Qbe(system.EType("envlink"))) if len(re) == 1 { vid, edge, success = re[0].Target, re[0], true } } return }
func (spec DataAlpha) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { // TODO this makes a loop...are we cool with that? success = true // impossible to fail here e = system.StdEdge{ Source: src.ID, Target: src.ID, Props: ps.NewMap(), EType: "data-provenance", } re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance"))) if len(re) == 1 { e = re[0] } return }
// utility func to create a StandardEdge. func mkEdge(id, source, target uint64, msgid uint64, etype string, props ...interface{}) system.StdEdge { e := system.StdEdge{ ID: id, Source: source, Target: target, EType: system.EType(etype), Props: ps.NewMap(), } var k string var v interface{} for len(props) > 1 { k, v, props = props[0].(string), props[1], props[2:] e.Props = e.Props.Set(k, system.Property{MsgSrc: msgid, Value: v}) } return e }
// RawMapToPropPMap fills a ps.Map with k/v pairs from the provided native Go map, // wrapping values in a types.Property struct using the provided msgid. // // If allowEmpty is false, only non-empty values will be included in the created map. func RawMapToPropPMap(msgid uint64, allowEmpty bool, in system.RawProps) ps.Map { m := ps.NewMap() var empty bool var err error if allowEmpty { for k, v := range in { m = m.Set(k, system.Property{MsgSrc: msgid, Value: v}) } } else { for k, v := range in { if empty, err = isEmptyValue(v); !empty && err == nil { m = m.Set(k, system.Property{MsgSrc: msgid, Value: v}) } } } return m }
// FillPropMap fills a ps.Map with the provided value pairs, wrapping values in a // types.Property struct using the provided msgid. // // If allowEmpty is false, only non-empty values will be included in the created map. func FillPropMap(msgid uint64, allowEmpty bool, p ...system.PropPair) ps.Map { m := ps.NewMap() var empty bool var err error if allowEmpty { for _, pair := range p { m = m.Set(pair.K, system.Property{MsgSrc: msgid, Value: pair.V}) } } else { for _, pair := range p { if empty, err = isEmptyValue(pair.V); !empty && err == nil { m = m.Set(pair.K, system.Property{MsgSrc: msgid, Value: pair.V}) } } } return m }
func (spec DataProvenance) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { // FIXME this presents another weird case where "success" is not binary. We *could* // find an already-existing data-provenance edge, but then have some net-addr params // change which cause it to fail to resolve to an environment. If we call that successful, // then it won't try to resolve again later...though, hm, just call it unsuccessful and // then try again one more time. Maybe it is fine. THINK IT THROUGH. e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "data-provenance", } e.Props = assignAddress(mid, spec.Address, e.Props, false) re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance"))) if len(re) == 1 { reresolve := maputil.AnyMatch(e.Props, re[0].Props, "hostname", "ipv4", "ipv6") e = re[0] if spec.SnapTime != "" { e.Props = e.Props.Set("snap-time", system.Property{MsgSrc: mid, Value: spec.SnapTime}) } if reresolve { e.Props = assignAddress(mid, spec.Address, e.Props, true) } else { return e, true } } envid, found := findEnvironment(g, e.Props) if !found { // TODO returning this already-modified edge necessitates that the core system // disregard 'failed' edges. which should be fine, that should be a guarantee return e, false } e.Target, success = findDataset(g, envid, spec.Dataset) return }
func getGraphFixture() *coreGraph { g := &coreGraph{vtuples: ps.NewMap(), vserial: 0} // Manually populate the graph with some dummy vertices and edges. // These don't necessarily line up with any real schemas, on purpose. // edge, id 10, connects vid 1 to vid 2. msgid 2. type "dummy-edge-type1". one prop - "eprop1": "foo". edge10 := mkEdge(10, 1, 2, 2, "dummy-edge-type1", "eprop1", "foo") // edge, id 11, connects vid 3 to vid 1. msgid 3. type "dummy-edge-type2". one prop - "eprop2": "bar". edge11 := mkEdge(11, 3, 1, 3, "dummy-edge-type2", "eprop2", "bar") // edge, id 12, connects vid 3 to vid 4. msgid 4. type "dummy-edge-type2". one prop - "eprop2": "baz". edge12 := mkEdge(12, 3, 4, 3, "dummy-edge-type2", "eprop2", "baz") // edge, id 13, connects vid 3 to vid 4. msgid 4. type "dummy-edge-type3". two props - "eprop2": "qux", "eprop3": 42. edge13 := mkEdge(13, 3, 4, 4, "dummy-edge-type3", "eprop2", "bar", "eprop3", 42) // vid 1, type "env". two props - "prop1": "bar", "prop2": 42. msgid 1 vt1 := mkTuple(1, system.NewVertex("env", 1, tprops("prop1", "foo", "prop2", 42)...), edge10, edge11) // one in, one out g.vtuples = g.vtuples.Set(strconv.Itoa(1), vt1) // vid 2, type "env". , "one prop - "prop1", "foo". msgid 2 vt2 := mkTuple(2, system.NewVertex("env", 2, tprops("prop1", "bar")...), edge10) // one in g.vtuples = g.vtuples.Set(strconv.Itoa(2), vt2) // vid 3, type "vt2". two props - "prop1", "bar", "bowser", "moo". msgid 3 vt3 := mkTuple(3, system.NewVertex("vt2", 3, tprops("prop1", "bar", "bowser", "moo")...), edge11, edge12, edge13) // three out g.vtuples = g.vtuples.Set(strconv.Itoa(3), vt3) // vid 4, type "vt3". three props - "prop1", "baz", "prop2", 42, "prop3", "qux". msgid 4 vt4 := mkTuple(4, system.NewVertex("vt3", 4, tprops("prop1", "baz", "prop2", 42, "prop3", "qux")...), edge12, edge13) // two in, same origin g.vtuples = g.vtuples.Set(strconv.Itoa(4), vt4) // vid 5, type "vt3". no props, no edges. msgid 5 vt5 := mkTuple(5, system.NewVertex("vt3", 5)) // none in or out g.vtuples = g.vtuples.Set(strconv.Itoa(5), vt5) g.vserial = 13 return g }
func BenchmarkMergeMessageOne(b *testing.B) { g := &coreGraph{vtuples: ps.NewMap()} for i := 0; i < b.N; i++ { g.Merge(0, msgs[0].UnificationForm()) } }
func (spec DataLink) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) { e = system.StdEdge{ Source: src.ID, Props: ps.NewMap(), EType: "datalink", } // DataLinks have a 'name' field that is expected to be unique for the source, if present if spec.Name != "" { // TODO 'name' is a traditional unique key; a change in it inherently denotes a new edge. how to handle this? // FIXME this approach just always updates the mid, which is weird? e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name}) re := g.OutWith(src.ID, q.Qbe(system.EType("datalink"), "name", spec.Name)) if len(re) == 1 { success = true e = re[0] } } if spec.Type != "" { e.Props = e.Props.Set("type", system.Property{MsgSrc: mid, Value: spec.Type}) } if spec.Subset != "" { e.Props = e.Props.Set("subset", system.Property{MsgSrc: mid, Value: spec.Subset}) } if spec.Interaction != "" { e.Props = e.Props.Set("interaction", system.Property{MsgSrc: mid, Value: spec.Interaction}) } // Special bits: if we have ConnUnix data, eliminate ConnNet data, and vice-versa. var isLocal bool if spec.ConnUnix.Path != "" { isLocal = true e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.ConnUnix.Path}) e.Props = e.Props.Delete("hostname") e.Props = e.Props.Delete("ipv4") e.Props = e.Props.Delete("ipv6") e.Props = e.Props.Delete("port") e.Props = e.Props.Delete("proto") } else { e.Props = e.Props.Set("port", system.Property{MsgSrc: mid, Value: spec.ConnNet.Port}) e.Props = e.Props.Set("proto", system.Property{MsgSrc: mid, Value: spec.ConnNet.Proto}) // can only be one of hostname, ipv4 or ipv6 if spec.ConnNet.Hostname != "" { e.Props = e.Props.Set("hostname", system.Property{MsgSrc: mid, Value: spec.ConnNet.Hostname}) } else if spec.ConnNet.Ipv4 != "" { e.Props = e.Props.Set("ipv4", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv4}) } else { e.Props = e.Props.Set("ipv6", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv6}) } } if success { return } var sock system.VertexTuple var rv system.VertexTupleVector // just for reuse // If net, must scan; if local, a bit easier. if !isLocal { // First, find the environment vertex rv = g.VerticesWith(q.Qbv(system.VType("environment"))) var envid uint64 for _, vt := range rv { if maputil.AnyMatch(e.Props, vt.Vertex.Properties, "hostname", "ipv4", "ipv6") { envid = vt.ID break } } // No matching env found, bail out if envid == 0 { return } // Now, walk the environment's edges to find the vertex representing the port rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "port", "port", spec.ConnNet.Port).And(q.Qbe(system.EType("envlink")))) if len(rv) != 1 { return } sock = rv[0] // With sock in hand, now find its proc rv = g.PredecessorsWith(sock.ID, q.Qbe(system.EType("listening"), "proto", spec.ConnNet.Proto).And(q.Qbv(system.VType("process")))) if len(rv) != 1 { // TODO could/will we ever allow >1? return } } else { envid, _, exists := findEnv(g, src) if !exists { // this is would be a pretty weird case return } // Walk the graph to find the vertex representing the unix socket rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "path", spec.ConnUnix.Path).And(q.Qbe(system.EType("envlink")))) if len(rv) != 1 { return } sock = rv[0] // With sock in hand, now find its proc rv = g.PredecessorsWith(sock.ID, q.Qbv(system.VType("process")).And(q.Qbe(system.EType("listening")))) if len(rv) != 1 { // TODO could/will we ever allow >1? return } } rv = g.SuccessorsWith(rv[0].ID, q.Qbv(system.VType("parent-dataset"))) // FIXME this absolutely could be more than 1 if len(rv) != 1 { return } dataset := rv[0] // if the spec indicates a subset, find it if spec.Subset != "" { rv = g.PredecessorsWith(rv[0].ID, q.Qbv(system.VType("dataset"), "name", spec.Subset).And(q.Qbe(system.EType("dataset-hierarchy")))) if len(rv) != 1 { return } dataset = rv[0] } // FIXME only recording the final target id is totally broken; see https://github.com/pipeviz/pipeviz/issues/37 // Aaaand we found our target. success = true e.Target = dataset.ID return }