Beispiel #1
0
func (spec specLocalLogic) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "logic-link",
	}

	// search for existing link
	re := g.OutWith(src.ID, q.Qbe(system.EType("logic-link"), "path", spec.Path))
	if len(re) == 1 {
		// TODO don't set the path prop again, it's the unique id...meh, same question here w/uniqueness as above
		success = true
		e = re[0]
		return
	}

	// no existing link found, search for proc directly
	envid, _, _ := findEnv(g, src)
	rv := g.PredecessorsWith(envid, q.Qbv(system.VType("logic-state"), "path", spec.Path))
	if len(rv) == 1 {
		success = true
		e.Target = rv[0].ID
	}

	return
}
Beispiel #2
0
func (spec specParentDataset) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "dataset-gateway",
	}
	e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})

	// check for existing link - there can be only be one
	re := g.OutWith(src.ID, q.Qbe(system.EType("dataset-gateway")))
	if len(re) == 1 {
		success = true
		e = re[0]
		// TODO semantics should preclude this from being able to change, but doing it dirty means force-setting it anyway for now
	} else {

		// no existing link found; search for proc directly
		envid, _, _ := findEnv(g, src)
		rv := g.PredecessorsWith(envid, q.Qbv(system.VType("parent-dataset"), "name", spec.Name))
		if len(rv) != 0 { // >1 shouldn't be possible
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Beispiel #3
0
func (spec EnvLink) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	_, e, success = findEnv(g, src)

	// Whether we find a match or not, have to merge in the EnvLink
	e.Props = maputil.FillPropMap(mid, false,
		pp("hostname", spec.Address.Hostname),
		pp("ipv4", spec.Address.Ipv4),
		pp("ipv6", spec.Address.Ipv6),
		pp("nick", spec.Nick),
	)

	// If we already found the matching edge, bail out now
	if success {
		return
	}

	rv := g.VerticesWith(q.Qbv(system.VType("environment")))
	for _, vt := range rv {
		// TODO this'll be cross-package eventually - reorg needed
		if maputil.AnyMatch(e.Props, vt.Vertex.Properties, "nick", "hostname", "ipv4", "ipv6") {
			success = true
			e.Target = vt.ID
			break
		}
	}

	return
}
Beispiel #4
0
func (spec specGitCommitParent) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "parent-commit",
	}

	re := g.OutWith(src.ID, q.Qbe(system.EType("parent-commit"), "pnum", spec.ParentNum))
	if len(re) > 0 {
		success = true
		e.Target = re[0].Target
		e.Props = re[0].Props
		// FIXME evidence of a problem here - since we're using pnum as the deduping identifier, there's no
		// way it could also sensibly change its MsgSrc value. This is very much a product of the intensional/extensional
		// identity problem: what does it mean to have the identifying data change? is it now a new thing? was it the old thing,
		// and it underwent a transition into the new thing? or is there no distinction between the old and new thing?
		e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
		e.ID = re[0].ID
	} else {
		rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
			e.Props = e.Props.Set("pnum", system.Property{MsgSrc: mid, Value: spec.ParentNum})
			e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})
		}
	}

	return
}
Beispiel #5
0
func (spec specUnixDomainListener) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// check for existing edge; this one is quite straightforward
	re := g.OutWith(src.ID, q.Qbe(system.EType("listening"), "type", "unix", "path", spec.Path))
	if len(re) == 1 {
		return re[0], true
	}

	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "listening",
	}

	e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.Path})

	envid, _, hasenv := findEnv(g, src)
	if hasenv {
		rv := g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "unix", "path", spec.Path))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Beispiel #6
0
func BenchmarkMergeMessageOneAndTwo(b *testing.B) {
	var g system.CoreGraph = &coreGraph{vtuples: ps.NewMap()}

	for i := 0; i < b.N; i++ {
		g.Merge(0, msgs[0].UnificationForm())
		g.Merge(0, msgs[1].UnificationForm())
	}
}
Beispiel #7
0
func commitUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	candidates := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", u.Vertex().Properties()["sha1"]))

	if len(candidates) > 0 { // there can be only one
		return candidates[0].ID
	}

	return 0
}
Beispiel #8
0
func parentDatasetUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	edge, success := u.ScopingSpecs()[0].(EnvLink).Resolve(g, 0, emptyVT(u.Vertex()))
	if !success {
		// FIXME scoping edge resolution failure does not mean no match - there could be an orphan
		return 0
	}

	props := u.Vertex().Properties()
	return findMatchingEnvId(g, edge, g.VerticesWith(q.Qbv(system.VType("parent-dataset"), "path", props["path"], "name", props["name"])))
}
Beispiel #9
0
func processUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	// only one scoping edge - the envlink
	edge, success := u.ScopingSpecs()[0].(EnvLink).Resolve(g, 0, emptyVT(u.Vertex()))
	if !success {
		// FIXME scoping edge resolution failure does not mean no match - there could be an orphan
		return 0
	}

	return findMatchingEnvId(g, edge, g.VerticesWith(q.Qbv(system.VType("process"), "pid", u.Vertex().Properties()["pid"])))
}
Beispiel #10
0
func findEnvironment(g system.CoreGraph, props ps.Map) (envid uint64, success bool) {
	rv := g.VerticesWith(q.Qbv(system.VType("environment")))
	for _, vt := range rv {
		if maputil.AnyMatch(props, vt.Vertex.Props(), "hostname", "ipv4", "ipv6", "nick") {
			return vt.ID, true
		}
	}

	return
}
Beispiel #11
0
func envUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	matches := g.VerticesWith(q.Qbv(system.VType("environment")))

	for _, e := range matches {
		if maputil.AnyMatch(e.Vertex.Properties, u.Vertex().Properties(), "hostname", "ipv4", "ipv6") {
			return e.ID
		}
	}

	return 0
}
Beispiel #12
0
func findMatchingEnvId(g system.CoreGraph, edge system.StdEdge, vtv system.VertexTupleVector) uint64 {
	for _, candidate := range vtv {
		for _, edge2 := range g.OutWith(candidate.ID, q.Qbe(system.EType("envlink"))) {
			if edge2.Target == edge.Target {
				return candidate.ID
			}
		}
	}

	return 0
}
Beispiel #13
0
func pkgYumUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	props := u.Vertex().Properties()
	vtv := g.VerticesWith(q.Qbv(system.VType("pkg-yum"),
		"name", props["name"],
		"version", props["version"],
		"arch", props["arch"],
		"epoch", props["epoch"],
	))

	if len(vtv) > 0 {
		return vtv[0].ID
	}
	return 0
}
Beispiel #14
0
func graphToJSON(g system.CoreGraph) ([]byte, error) {
	var vertices []interface{}
	for _, v := range g.VerticesWith(q.Qbv(system.VTypeNone)) {
		vertices = append(vertices, v.Flat())
	}

	// TODO use something that lets us write to a reusable byte buffer instead
	return json.Marshal(struct {
		Id       uint64        `json:"id"`
		Vertices []interface{} `json:"vertices"`
	}{
		Id:       g.MsgID(),
		Vertices: vertices,
	})
}
Beispiel #15
0
// Searches the given vertex's out-edges to find its environment's vertex id.
//
// Also conveniently initializes a StandardEdge to the standard zero-state for an envlink.
func findEnv(g system.CoreGraph, vt system.VertexTuple) (vid uint64, edge system.StdEdge, success bool) {
	edge = system.StdEdge{
		Source: vt.ID,
		Props:  ps.NewMap(),
		EType:  "envlink",
	}

	if vt.ID != 0 {
		re := g.OutWith(vt.ID, q.Qbe(system.EType("envlink")))
		if len(re) == 1 {
			vid, edge, success = re[0].Target, re[0], true
		}
	}

	return
}
Beispiel #16
0
func (spec DataAlpha) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// TODO this makes a loop...are we cool with that?
	success = true // impossible to fail here
	e = system.StdEdge{
		Source: src.ID,
		Target: src.ID,
		Props:  ps.NewMap(),
		EType:  "data-provenance",
	}

	re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
	if len(re) == 1 {
		e = re[0]
	}

	return
}
Beispiel #17
0
func datasetUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	vtv := g.VerticesWith(q.Qbv(system.VType("dataset"), "name", u.Vertex().Properties()["name"]))
	if len(vtv) == 0 {
		return 0
	}

	spec := u.ScopingSpecs()[0].(specDatasetHierarchy)
	el, success := spec.Environment.Resolve(g, 0, emptyVT(u.Vertex()))
	// FIXME scoping edge resolution failure does not mean no match - there could be an orphan
	if success {
		for _, vt := range vtv {
			if id := findMatchingEnvId(g, el, g.SuccessorsWith(vt.ID, q.Qbe(system.EType("dataset-hierarchy")))); id != 0 {
				return vt.ID
			}
		}
	}

	return 0
}
Beispiel #18
0
func findDataset(g system.CoreGraph, envid uint64, name []string) (id uint64, success bool) {
	// first time through use the parent type
	vtype := system.VType("parent-dataset")
	id = envid

	var n string
	for len(name) > 0 {
		n, name = name[0], name[1:]
		rv := g.PredecessorsWith(id, q.Qbv(vtype, "name", n))
		vtype = "dataset"

		if len(rv) != 1 {
			return 0, false
		}

		id = rv[0].ID
	}

	return id, true
}
Beispiel #19
0
func (spec DataProvenance) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	// FIXME this presents another weird case where "success" is not binary. We *could*
	// find an already-existing data-provenance edge, but then have some net-addr params
	// change which cause it to fail to resolve to an environment. If we call that successful,
	// then it won't try to resolve again later...though, hm, just call it unsuccessful and
	// then try again one more time. Maybe it is fine. THINK IT THROUGH.

	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "data-provenance",
	}
	e.Props = assignAddress(mid, spec.Address, e.Props, false)

	re := g.OutWith(src.ID, q.Qbe(system.EType("data-provenance")))
	if len(re) == 1 {
		reresolve := maputil.AnyMatch(e.Props, re[0].Props, "hostname", "ipv4", "ipv6")

		e = re[0]
		if spec.SnapTime != "" {
			e.Props = e.Props.Set("snap-time", system.Property{MsgSrc: mid, Value: spec.SnapTime})
		}

		if reresolve {
			e.Props = assignAddress(mid, spec.Address, e.Props, true)
		} else {
			return e, true
		}
	}

	envid, found := findEnvironment(g, e.Props)
	if !found {
		// TODO returning this already-modified edge necessitates that the core system
		// disregard 'failed' edges. which should be fine, that should be a guarantee
		return e, false
	}

	e.Target, success = findDataset(g, envid, spec.Dataset)
	return
}
Beispiel #20
0
func (spec specCommit) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "version",
	}
	e.Props = e.Props.Set("sha1", system.Property{MsgSrc: mid, Value: spec.Sha1})

	re := g.OutWith(src.ID, q.Qbe(system.EType("version")))
	if len(re) > 0 {
		sha1, _ := re[0].Props.Lookup("sha1")
		e.ID = re[0].ID // FIXME setting the id to non-0 AND failing is currently unhandled
		if sha1.(system.Property).Value == spec.Sha1 {
			success = true
			e.Target = re[0].Target
		} else {
			rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
			if len(rv) == 1 {
				success = true
				e.Target = rv[0].ID
			}
		}
	} else {
		rv := g.VerticesWith(q.Qbv(system.VType("commit"), "sha1", spec.Sha1))
		if len(rv) == 1 {
			success = true
			e.Target = rv[0].ID
		}
	}

	return
}
Beispiel #21
0
func commUnify(g system.CoreGraph, u system.UnifyInstructionForm) uint64 {
	// only one scoping edge - the envlink
	edge, success := u.ScopingSpecs()[0].(EnvLink).Resolve(g, 0, emptyVT(u.Vertex()))
	if !success {
		// FIXME scoping edge resolution failure does not mean no match - there could be an orphan
		return 0
	}

	vp := u.Vertex().Properties()
	typ, _ := vp["type"]
	path, haspath := vp["path"]
	if haspath {
		return findMatchingEnvId(g, edge, g.VerticesWith(q.Qbv(system.VType("comm"),
			"type", typ,
			"path", path)))
	} else {
		port, _ := vp["port"]
		return findMatchingEnvId(g, edge, g.VerticesWith(q.Qbv(system.VType("comm"),
			"type", typ,
			"port", port)))
	}
}
Beispiel #22
0
func init() {
	startTime := time.Now().UTC()
	expvar.Publish("Uptime", expvar.Func(func() interface{} { return int64(time.Since(startTime)) }))

	expvar.Publish("Goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() }))

	// subscribe to the broker in order to report data about current graph
	c := broker.Get().Subscribe()

	// Instantiate a real, empty graph to ensure the interface type is never nil when it might be called
	var g system.CoreGraph = represent.NewGraph()
	go func() {
		for latest := range c {
			g = latest
		}
	}()
	expvar.Publish("MsgId", expvar.Func(func() interface{} { return g.MsgId() }))

	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()
}
Beispiel #23
0
// Generates a .dot-format representation of the given CoreGraph, suitable for
// rendering into output by graphviz (or other utilities).
func GenerateDot(g system.CoreGraph) []byte {
	buf := new(bytes.Buffer)

	// begin the graph
	buf.WriteString("digraph{\n")
	buf.WriteString("fontsize=16")

	// first, write all vertices
	for _, v := range g.VerticesWith(q.Qbv()) {
		lbltype := "label"
		var props string
		switch v.Vertex.Typ() {
		case "environment":
			props = "\tshape=house,style=filled,fillcolor=orange,fontsize=20\n"
		case "logic-state":
			props = "\tshape=box3d,style=filled,fillcolor=purple,fontcolor=white,fontsize=18,\n"
		case "process":
			props = "\tshape=oval,style=filled,fillcolor=green,\n"
		case "dataset", "parent-dataset":
			props = "\tshape=folder,style=filled,fillcolor=brown,fontcolor=white,\n"
		case "comm":
			props = "\tshape=doubleoctagon,style=filled,fillcolor=cyan,\n"
		case "git-commit":
			props = "\tshape=box,style=filled,fillcolor=grey\n"
		case "git-tag", "git-branch":
			props = "\tshape=cds,margin=\"0.22,0.22\",\n"
		case "test-result":
			props = "\tshape=note\n"
		}

		buf.WriteString(fmt.Sprintf(
			"\t\"v%d\" [%s%s=\"id: %d\nvtype: %s",
			v.ID, props, lbltype, v.ID, v.Vertex.Typ()))

		v.Vertex.Props().ForEach(func(k string, val ps.Any) {
			prop := val.(system.Property)
			var format string
			switch pv := prop.Value.(type) {
			case []byte, [20]byte:
				format = "%x"
			case int, int64, int32, int16, int8, uint, uint64, uint32, uint16, uint8:
				format = "%d"
			case string:
				prop.Value = strings.Trim(strconv.QuoteToASCII(pv), `"`)
				format = "%s"
			default:
				format = "%s"
			}
			buf.WriteString(fmt.Sprintf(
				"\n%s: "+format+" (%d)",
				k, prop.Value, prop.MsgSrc))
		})

		buf.WriteString("\"\n")

		buf.WriteString("];\n")
	}

	// pass through a second time to write all edges
	for _, v := range g.VerticesWith(q.Qbv()) {
		v.OutEdges.ForEach(func(k string, val ps.Any) {
			edge := val.(system.StdEdge)
			buf.WriteString(fmt.Sprintf(
				"\t\"v%d\" -> \"v%d\" [\n\tlabel=\"id: %d\netype: %s",
				edge.Source, edge.Target, edge.ID, edge.EType))

			edge.Props.ForEach(func(k2 string, val2 ps.Any) {
				prop := val2.(system.Property)
				var format string
				switch pv := prop.Value.(type) {
				case []byte:
					format = "%x"
				case int, int64, int32, int16, int8, uint, uint64, uint32, uint16, uint8:
					format = "%d"
				case string:
					prop.Value = strings.Trim(strconv.QuoteToASCII(pv), `"`)
					format = "%s"
				default:
					format = "%s"
				}
				buf.WriteString(fmt.Sprintf(
					"\n%s: "+format+" (%d)",
					k2, prop.Value, prop.MsgSrc))
			})

			buf.WriteString("\"\n")

			switch edge.EType {
			case "envlink":
				buf.WriteString("\tstyle=dashed\n")
			}

			buf.WriteString("];\n")
		})
	}

	// close out the graph
	buf.WriteString("}\n")

	return buf.Bytes()
}
Beispiel #24
0
func (spec DataLink) Resolve(g system.CoreGraph, mid uint64, src system.VertexTuple) (e system.StdEdge, success bool) {
	e = system.StdEdge{
		Source: src.ID,
		Props:  ps.NewMap(),
		EType:  "datalink",
	}

	// DataLinks have a 'name' field that is expected to be unique for the source, if present
	if spec.Name != "" {
		// TODO 'name' is a traditional unique key; a change in it inherently denotes a new edge. how to handle this?
		// FIXME this approach just always updates the mid, which is weird?
		e.Props = e.Props.Set("name", system.Property{MsgSrc: mid, Value: spec.Name})

		re := g.OutWith(src.ID, q.Qbe(system.EType("datalink"), "name", spec.Name))
		if len(re) == 1 {
			success = true
			e = re[0]
		}
	}

	if spec.Type != "" {
		e.Props = e.Props.Set("type", system.Property{MsgSrc: mid, Value: spec.Type})
	}
	if spec.Subset != "" {
		e.Props = e.Props.Set("subset", system.Property{MsgSrc: mid, Value: spec.Subset})
	}
	if spec.Interaction != "" {
		e.Props = e.Props.Set("interaction", system.Property{MsgSrc: mid, Value: spec.Interaction})
	}

	// Special bits: if we have ConnUnix data, eliminate ConnNet data, and vice-versa.
	var isLocal bool
	if spec.ConnUnix.Path != "" {
		isLocal = true
		e.Props = e.Props.Set("path", system.Property{MsgSrc: mid, Value: spec.ConnUnix.Path})
		e.Props = e.Props.Delete("hostname")
		e.Props = e.Props.Delete("ipv4")
		e.Props = e.Props.Delete("ipv6")
		e.Props = e.Props.Delete("port")
		e.Props = e.Props.Delete("proto")
	} else {
		e.Props = e.Props.Set("port", system.Property{MsgSrc: mid, Value: spec.ConnNet.Port})
		e.Props = e.Props.Set("proto", system.Property{MsgSrc: mid, Value: spec.ConnNet.Proto})

		// can only be one of hostname, ipv4 or ipv6
		if spec.ConnNet.Hostname != "" {
			e.Props = e.Props.Set("hostname", system.Property{MsgSrc: mid, Value: spec.ConnNet.Hostname})
		} else if spec.ConnNet.Ipv4 != "" {
			e.Props = e.Props.Set("ipv4", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv4})
		} else {
			e.Props = e.Props.Set("ipv6", system.Property{MsgSrc: mid, Value: spec.ConnNet.Ipv6})
		}
	}

	if success {
		return
	}

	var sock system.VertexTuple
	var rv system.VertexTupleVector // just for reuse
	// If net, must scan; if local, a bit easier.
	if !isLocal {
		// First, find the environment vertex
		rv = g.VerticesWith(q.Qbv(system.VType("environment")))
		var envid uint64
		for _, vt := range rv {
			if maputil.AnyMatch(e.Props, vt.Vertex.Properties, "hostname", "ipv4", "ipv6") {
				envid = vt.ID
				break
			}
		}

		// No matching env found, bail out
		if envid == 0 {
			return
		}

		// Now, walk the environment's edges to find the vertex representing the port
		rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "type", "port", "port", spec.ConnNet.Port).And(q.Qbe(system.EType("envlink"))))

		if len(rv) != 1 {
			return
		}
		sock = rv[0]

		// With sock in hand, now find its proc
		rv = g.PredecessorsWith(sock.ID, q.Qbe(system.EType("listening"), "proto", spec.ConnNet.Proto).And(q.Qbv(system.VType("process"))))
		if len(rv) != 1 {
			// TODO could/will we ever allow >1?
			return
		}
	} else {
		envid, _, exists := findEnv(g, src)

		if !exists {
			// this is would be a pretty weird case
			return
		}

		// Walk the graph to find the vertex representing the unix socket
		rv = g.PredecessorsWith(envid, q.Qbv(system.VType("comm"), "path", spec.ConnUnix.Path).And(q.Qbe(system.EType("envlink"))))
		if len(rv) != 1 {
			return
		}
		sock = rv[0]

		// With sock in hand, now find its proc
		rv = g.PredecessorsWith(sock.ID, q.Qbv(system.VType("process")).And(q.Qbe(system.EType("listening"))))
		if len(rv) != 1 {
			// TODO could/will we ever allow >1?
			return
		}
	}

	rv = g.SuccessorsWith(rv[0].ID, q.Qbv(system.VType("parent-dataset")))
	// FIXME this absolutely could be more than 1
	if len(rv) != 1 {
		return
	}
	dataset := rv[0]

	// if the spec indicates a subset, find it
	if spec.Subset != "" {
		rv = g.PredecessorsWith(rv[0].ID, q.Qbv(system.VType("dataset"), "name", spec.Subset).And(q.Qbe(system.EType("dataset-hierarchy"))))
		if len(rv) != 1 {
			return
		}
		dataset = rv[0]
	}

	// FIXME only recording the final target id is totally broken; see https://github.com/pipeviz/pipeviz/issues/37

	// Aaaand we found our target.
	success = true
	e.Target = dataset.ID
	return
}