Exemple #1
0
// TODO it would be better to not have to have this here, at all.
func emptyVT(v system.ProtoVertex) system.VertexTuple {
	var props []system.PropPair
	for k, v := range v.Properties() {
		props = append(props, pp(k, v))
	}

	return system.VertexTuple{
		Vertex:   system.NewVertex(v.Type(), 0, props...),
		InEdges:  ps.NewMap(),
		OutEdges: ps.NewMap(),
	}
}
Exemple #2
0
func (spec specGitCommitParent) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "parent-commit",
	}

	re := g.OutWith(src.ID, q.Qbe(system.EType("parent-commit"), "pnum", spec.ParentNum))
	if len(re) > 0 {
		success = true
		e.Target = re[0].Target
		e.Props = re[0].Props
		// FIXME evidence of a problem here - since we're using pnum as the deduping identifier, there's no
		// way it could also sensibly change its MsgSrc value. This is very much a product of the intensional/extensional
		// identity problem: what does it mean to have the identifying data change? is it now a new thing? was it the old thing,
		// and it underwent a transition into the new thing? or is there no distinction between the old and new thing?
		e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
		e.ID = re[0].ID
	} else {
		rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
			e.Props = e.Props.Set("pnum", system.Property{MsgSrc: mid, Value: spec.ParentNum})
			e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
		}
	}

	return
}
Exemple #3
0
func (spec specLocalLogic) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "logic-link",
	}

	// search for existing link
	re := g.OutWith(src.ID, q.Qbe(system.EType("logic-link"), "path", spec.Path))
	if len(re) == 1 {
		// TODO don't set the path prop again, it's the unique id...meh, same question here w/uniqueness as above
		success = true
		e = re[0]
		return
	}

	// no existing link found, search for proc directly
	envid, _, _ := findEnv(g, src)
	rv := g.PredecessorsWith(envid, q.Qbv(system.VType("logic-state"), "path", spec.Path))
	if len(rv) == 1 {
		success = true
		e.Target = rv[0].ID
	}

	return
}
Exemple #4
0
func (spec specParentDataset) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "dataset-gateway",
	}
	e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})

	// check for existing link - there can be only be one
	re := g.OutWith(src.ID, q.Qbe(system.EType("dataset-gateway")))
	if len(re) == 1 {
		success = true
		e = re[0]
		// TODO semantics should preclude this from being able to change, but doing it dirty means force-setting it anyway for now
	} else {

		// no existing link found; search for proc directly
		envid, _, _ := findEnv(g, src)
		rv := g.PredecessorsWith(envid, q.Qbv(system.VType("parent-dataset"), "name", spec.Name))
		if len(rv) != 0 { // >1 shouldn't be possible
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Exemple #5
0
func (spec specUnixDomainListener) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// check for existing edge; this one is quite straightforward
	re := g.OutWith(src.ID, q.Qbe(system.EType("listening"), "type", "unix", "path", spec.Path))
	if len(re) == 1 {
		return re[0], true
	}

	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "listening",
	}

	e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.Path})

	envid, _, hasenv := findEnv(g, src)
	if hasenv {
		rv := g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "unix", "path", spec.Path))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Exemple #6
0
func (spec specCommit) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "version",
	}
	e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})

	re := g.OutWith(src.ID, q.Qbe(system.EType("version")))
	if len(re) > 0 {
		sha1, _ := re[0].Props.Lookup("sha1")
		e.ID = re[0].ID // FIXME setting the id to non-0 AND failing is currently unhandled
		if sha1.(system.Property).Value == spec.Sha1 {
			success = true
			e.Target = re[0].Target
		} else {
			rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
			if len(rv) == 1 {
				success = true
				e.Target = rv[0].ID
			}
		}
	} else {
		rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Exemple #7
0
func BenchmarkMergeMessageOneAndTwo(b *testing.B) {
	var g system.CoreGraph = &coreGraph{vtuples: ps.NewMap()}

	for i := 0; i < b.N; i++ {
		g.Merge(0, msgs[0].UnificationForm())
		g.Merge(0, msgs[1].UnificationForm())
	}
}
Exemple #8
0
// Transform a slice of kv pairs into a map.
func mapPairs(pairs ...p) (m ps.Map) {
	m = ps.NewMap()
	for _, kv := range pairs {
		m = m.Set(kv.K, kv.V)
	}

	return
}
Exemple #9
0
// utility func to create a vtTuple. puts edges in the right place by
// checking source/target ids. panics if they don't line up!
func mkTuple(vid uint64, vtx system.StdVertex, edges ...system.StdEdge) system.VertexTuple {
	vt := system.VertexTuple{
		ID:       vid,
		Vertex:   vtx,
		InEdges:  ps.NewMap(),
		OutEdges: ps.NewMap(),
	}

	for _, e := range edges {
		if e.Source == vid {
			vt.OutEdges = vt.OutEdges.Set(strconv.FormatUint(e.ID, 10), e)
		} else if e.Target == vid {
			vt.InEdges = vt.InEdges.Set(strconv.FormatUint(e.ID, 10), e)
		} else {
			panic("edge had neither source nor target of vid")
		}
	}

	return vt
}
Exemple #10
0
func fillMapIgnoreZero(msgid uint64, p ...PropPair) ps.Map {
	m := ps.NewMap()
	var zero bool
	var err error

	for _, pair := range p {
		if zero, err = isZero(pair.V); !zero && err == nil {
			m = m.Set(pair.K, Property{MsgSrc: msgid, Value: pair.V})
		}
	}

	return m
}
Exemple #11
0
func TestClone(t *testing.T) {
	g := &coreGraph{vtuples: ps.NewMap(), vserial: 0}
	g.vserial = 2
	g.vtuples = g.vtuples.Set("foo", "val")

	var g2 *coreGraph = g.clone()
	g2.vserial = 4
	g2.vtuples = g2.vtuples.Set("foo", "newval")

	if g.vserial != 2 {
		t.Errorf("changes in cloned graph propagated back to original")
	}
	if val, _ := g2.vtuples.Lookup("foo"); val != "newval" {
		t.Errorf("map somehow propagated changes back up to original map")
	}
}
Exemple #12
0
// Searches the given vertex's out-edges to find its environment's vertex id.
//
// Also conveniently initializes a StandardEdge to the standard zero-state for an envlink.
func findEnv(g system.CoreGraph, vt system.VertexTuple) (vid uint64, edge system.StdEdge, success bool) {
	edge = system.StdEdge{
		Source: vt.ID,
		Props:  ps.NewMap(),
		EType:  "envlink",
	}

	if vt.ID != 0 {
		re := g.OutWith(vt.ID, q.Qbe(system.EType("envlink")))
		if len(re) == 1 {
			vid, edge, success = re[0].Target, re[0], true
		}
	}

	return
}
Exemple #13
0
func (spec DataAlpha) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// TODO this makes a loop...are we cool with that?
	success = true // impossible to fail here
	e = system.StdEdge{
		Source: src.ID,
		Target: src.ID,
		Props:  ps.NewMap(),
		EType:  "data-provenance",
	}

	re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
	if len(re) == 1 {
		e = re[0]
	}

	return
}
Exemple #14
0
// utility func to create a StandardEdge.
func mkEdge(id, source, target uint64, msgid uint64, etype string, props ...interface{}) system.StdEdge {
	e := system.StdEdge{
		ID:     id,
		Source: source,
		Target: target,
		EType:  system.EType(etype),
		Props:  ps.NewMap(),
	}

	var k string
	var v interface{}
	for len(props) > 1 {
		k, v, props = props[0].(string), props[1], props[2:]
		e.Props = e.Props.Set(k, system.Property{MsgSrc: msgid, Value: v})
	}

	return e
}
Exemple #15
0
// RawMapToPropPMap fills a ps.Map with k/v pairs from the provided native Go map,
// wrapping values in a types.Property struct using the provided msgid.
//
// If allowEmpty is false, only non-empty values will be included in the created map.
func RawMapToPropPMap(msgid uint64, allowEmpty bool, in system.RawProps) ps.Map {
	m := ps.NewMap()
	var empty bool
	var err error

	if allowEmpty {
		for k, v := range in {
			m = m.Set(k, system.Property{MsgSrc: msgid, Value: v})
		}
	} else {
		for k, v := range in {
			if empty, err = isEmptyValue(v); !empty && err == nil {
				m = m.Set(k, system.Property{MsgSrc: msgid, Value: v})
			}
		}
	}

	return m
}
Exemple #16
0
// FillPropMap fills a ps.Map with the provided value pairs, wrapping values in a
// types.Property struct using the provided msgid.
//
// If allowEmpty is false, only non-empty values will be included in the created map.
func FillPropMap(msgid uint64, allowEmpty bool, p ...system.PropPair) ps.Map {
	m := ps.NewMap()
	var empty bool
	var err error

	if allowEmpty {
		for _, pair := range p {
			m = m.Set(pair.K, system.Property{MsgSrc: msgid, Value: pair.V})
		}
	} else {
		for _, pair := range p {
			if empty, err = isEmptyValue(pair.V); !empty && err == nil {
				m = m.Set(pair.K, system.Property{MsgSrc: msgid, Value: pair.V})
			}
		}
	}

	return m
}
Exemple #17
0
func (spec DataProvenance) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// FIXME this presents another weird case where "success" is not binary. We *could*
	// find an already-existing data-provenance edge, but then have some net-addr params
	// change which cause it to fail to resolve to an environment. If we call that successful,
	// then it won't try to resolve again later...though, hm, just call it unsuccessful and
	// then try again one more time. Maybe it is fine. THINK IT THROUGH.

	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "data-provenance",
	}
	e.Props = assignAddress(mid, spec.Address, e.Props, false)

	re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
	if len(re) == 1 {
		reresolve := maputil.AnyMatch(e.Props, re[0].Props, "hostname", "ipv4", "ipv6")

		e = re[0]
		if spec.SnapTime != "" {
			e.Props = e.Props.Set("snap-time", system.Property{MsgSrc: mid, Value: spec.SnapTime})
		}

		if reresolve {
			e.Props = assignAddress(mid, spec.Address, e.Props, true)
		} else {
			return e, true
		}
	}

	envid, found := findEnvironment(g, e.Props)
	if !found {
		// TODO returning this already-modified edge necessitates that the core system
		// disregard 'failed' edges. which should be fine, that should be a guarantee
		return e, false
	}

	e.Target, success = findDataset(g, envid, spec.Dataset)
	return
}
Exemple #18
0
func getGraphFixture() *coreGraph {
	g := &coreGraph{vtuples: ps.NewMap(), vserial: 0}

	// Manually populate the graph with some dummy vertices and edges.
	// These don't necessarily line up with any real schemas, on purpose.

	// edge, id 10, connects vid 1 to vid 2. msgid 2. type "dummy-edge-type1". one prop - "eprop1": "foo".
	edge10 := mkEdge(10, 1, 2, 2, "dummy-edge-type1", "eprop1", "foo")
	// edge, id 11, connects vid 3 to vid 1. msgid 3. type "dummy-edge-type2". one prop - "eprop2": "bar".
	edge11 := mkEdge(11, 3, 1, 3, "dummy-edge-type2", "eprop2", "bar")
	// edge, id 12, connects vid 3 to vid 4. msgid 4. type "dummy-edge-type2". one prop - "eprop2": "baz".
	edge12 := mkEdge(12, 3, 4, 3, "dummy-edge-type2", "eprop2", "baz")
	// edge, id 13, connects vid 3 to vid 4. msgid 4. type "dummy-edge-type3". two props - "eprop2": "qux", "eprop3": 42.
	edge13 := mkEdge(13, 3, 4, 4, "dummy-edge-type3", "eprop2", "bar", "eprop3", 42)

	// vid 1, type "env". two props - "prop1": "bar", "prop2": 42. msgid 1
	vt1 := mkTuple(1, system.NewVertex("env", 1, tprops("prop1", "foo", "prop2", 42)...), edge10, edge11) // one in, one out
	g.vtuples = g.vtuples.Set(strconv.Itoa(1), vt1)

	// vid 2, type "env". , "one prop - "prop1", "foo". msgid 2
	vt2 := mkTuple(2, system.NewVertex("env", 2, tprops("prop1", "bar")...), edge10) // one in
	g.vtuples = g.vtuples.Set(strconv.Itoa(2), vt2)

	// vid 3, type "vt2". two props - "prop1", "bar", "bowser", "moo". msgid 3
	vt3 := mkTuple(3, system.NewVertex("vt2", 3, tprops("prop1", "bar", "bowser", "moo")...), edge11, edge12, edge13) // three out
	g.vtuples = g.vtuples.Set(strconv.Itoa(3), vt3)

	// vid 4, type "vt3". three props - "prop1", "baz", "prop2", 42, "prop3", "qux". msgid 4
	vt4 := mkTuple(4, system.NewVertex("vt3", 4, tprops("prop1", "baz", "prop2", 42, "prop3", "qux")...), edge12, edge13) // two in, same origin
	g.vtuples = g.vtuples.Set(strconv.Itoa(4), vt4)

	// vid 5, type "vt3". no props, no edges. msgid 5
	vt5 := mkTuple(5, system.NewVertex("vt3", 5)) // none in or out
	g.vtuples = g.vtuples.Set(strconv.Itoa(5), vt5)
	g.vserial = 13

	return g
}
Exemple #19
0
func BenchmarkMergeMessageOne(b *testing.B) {
	g := &coreGraph{vtuples: ps.NewMap()}
	for i := 0; i < b.N; i++ {
		g.Merge(0, msgs[0].UnificationForm())
	}
}
Exemple #20
0
func (spec DataLink) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "datalink",
	}

	// DataLinks have a 'name' field that is expected to be unique for the source, if present
	if spec.Name != "" {
		// TODO 'name' is a traditional unique key; a change in it inherently denotes a new edge. how to handle this?
		// FIXME this approach just always updates the mid, which is weird?
		e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})

		re := g.OutWith(src.ID, q.Qbe(system.EType("datalink"), "name", spec.Name))
		if len(re) == 1 {
			success = true
			e = re[0]
		}
	}

	if spec.Type != "" {
		e.Props = e.Props.Set("type", system.Property{MsgSrc: mid, Value: spec.Type})
	}
	if spec.Subset != "" {
		e.Props = e.Props.Set("subset", system.Property{MsgSrc: mid, Value: spec.Subset})
	}
	if spec.Interaction != "" {
		e.Props = e.Props.Set("interaction", system.Property{MsgSrc: mid, Value: spec.Interaction})
	}

	// Special bits: if we have ConnUnix data, eliminate ConnNet data, and vice-versa.
	var isLocal bool
	if spec.ConnUnix.Path != "" {
		isLocal = true
		e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.ConnUnix.Path})
		e.Props = e.Props.Delete("hostname")
		e.Props = e.Props.Delete("ipv4")
		e.Props = e.Props.Delete("ipv6")
		e.Props = e.Props.Delete("port")
		e.Props = e.Props.Delete("proto")
	} else {
		e.Props = e.Props.Set("port", system.Property{MsgSrc: mid, Value: spec.ConnNet.Port})
		e.Props = e.Props.Set("proto", system.Property{MsgSrc: mid, Value: spec.ConnNet.Proto})

		// can only be one of hostname, ipv4 or ipv6
		if spec.ConnNet.Hostname != "" {
			e.Props = e.Props.Set("hostname", system.Property{MsgSrc: mid, Value: spec.ConnNet.Hostname})
		} else if spec.ConnNet.Ipv4 != "" {
			e.Props = e.Props.Set("ipv4", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv4})
		} else {
			e.Props = e.Props.Set("ipv6", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv6})
		}
	}

	if success {
		return
	}

	var sock system.VertexTuple
	var rv system.VertexTupleVector // just for reuse
	// If net, must scan; if local, a bit easier.
	if !isLocal {
		// First, find the environment vertex
		rv = g.VerticesWith(q.Qbv(system.VType("environment")))
		var envid uint64
		for _, vt := range rv {
			if maputil.AnyMatch(e.Props, vt.Vertex.Properties, "hostname", "ipv4", "ipv6") {
				envid = vt.ID
				break
			}
		}

		// No matching env found, bail out
		if envid == 0 {
			return
		}

		// Now, walk the environment's edges to find the vertex representing the port
		rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "port", "port", spec.ConnNet.Port).And(q.Qbe(system.EType("envlink"))))

		if len(rv) != 1 {
			return
		}
		sock = rv[0]

		// With sock in hand, now find its proc
		rv = g.PredecessorsWith(sock.ID, q.Qbe(system.EType("listening"), "proto", spec.ConnNet.Proto).And(q.Qbv(system.VType("process"))))
		if len(rv) != 1 {
			// TODO could/will we ever allow >1?
			return
		}
	} else {
		envid, _, exists := findEnv(g, src)

		if !exists {
			// this is would be a pretty weird case
			return
		}

		// Walk the graph to find the vertex representing the unix socket
		rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "path", spec.ConnUnix.Path).And(q.Qbe(system.EType("envlink"))))
		if len(rv) != 1 {
			return
		}
		sock = rv[0]

		// With sock in hand, now find its proc
		rv = g.PredecessorsWith(sock.ID, q.Qbv(system.VType("process")).And(q.Qbe(system.EType("listening"))))
		if len(rv) != 1 {
			// TODO could/will we ever allow >1?
			return
		}
	}

	rv = g.SuccessorsWith(rv[0].ID, q.Qbv(system.VType("parent-dataset")))
	// FIXME this absolutely could be more than 1
	if len(rv) != 1 {
		return
	}
	dataset := rv[0]

	// if the spec indicates a subset, find it
	if spec.Subset != "" {
		rv = g.PredecessorsWith(rv[0].ID, q.Qbv(system.VType("dataset"), "name", spec.Subset).And(q.Qbe(system.EType("dataset-hierarchy"))))
		if len(rv) != 1 {
			return
		}
		dataset = rv[0]
	}

	// FIXME only recording the final target id is totally broken; see https://github.com/pipeviz/pipeviz/issues/37

	// Aaaand we found our target.
	success = true
	e.Target = dataset.ID
	return
}