Example #1
0
func addReverseEdges(ctx context.Context, gs graphstore.Service) error {
	log.Println("Adding reverse edges")
	var (
		totalEntries int
		addedEdges   int
	)
	startTime := time.Now()
	err := gs.Scan(ctx, new(spb.ScanRequest), func(entry *spb.Entry) error {
		kind := entry.EdgeKind
		if kind != "" && edges.IsForward(kind) {
			if err := gs.Write(ctx, &spb.WriteRequest{
				Source: entry.Target,
				Update: []*spb.WriteRequest_Update{{
					Target:    entry.Source,
					EdgeKind:  edges.Mirror(kind),
					FactName:  entry.FactName,
					FactValue: entry.FactValue,
				}},
			}); err != nil {
				return fmt.Errorf("Failed to write reverse edge: %v", err)
			}
			addedEdges++
		}
		totalEntries++
		return nil
	})
	log.Printf("Wrote %d reverse edges to GraphStore (%d total entries): %v", addedEdges, totalEntries, time.Since(startTime))
	return err
}
Example #2
0
// CrossReference returns a (Referent, TargetAnchor) *ipb.CrossReference
// equivalent to the given decoration.  The decoration's anchor is expanded
// given its parent file and associated Normalizer.
func CrossReference(file *srvpb.File, norm *xrefs.Normalizer, d *srvpb.FileDecorations_Decoration, tgt *srvpb.Node) (*ipb.CrossReference, error) {
	if file == nil || norm == nil {
		return nil, errors.New("missing decoration's parent file")
	}

	ea, err := ExpandAnchor(d.Anchor, file, norm, edges.Mirror(d.Kind))
	if err != nil {
		return nil, fmt.Errorf("error expanding anchor {%+v}: %v", d.Anchor, err)
	}
	// Throw away most of the referent's facts.  They are not needed.
	var selected []*cpb.Fact
	if tgt != nil {
		for _, fact := range tgt.Fact {
			if fact.Name == facts.Complete {
				selected = append(selected, fact)
			}
		}
	}
	return &ipb.CrossReference{
		Referent: &srvpb.Node{
			Ticket: d.Target,
			Fact:   selected,
		},
		TargetAnchor: ea,
	}, nil
}
Example #3
0
// EnsureReverseEdges checks if gs contains reverse edges.  If it doesn't, it
// will scan gs for all forward edges, adding a reverse for each back into the
// GraphStore.  This is necessary for a GraphStoreService to work properly.
func EnsureReverseEdges(ctx context.Context, gs graphstore.Service) error {
	var edge *spb.Entry
	if err := gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
		if graphstore.IsEdge(e) {
			edge = e
			return io.EOF
		}
		return nil
	}); err != nil {
		return err
	}

	if edge == nil {
		log.Println("No edges found in GraphStore")
		return nil
	} else if edges.IsReverse(edge.EdgeKind) {
		return nil
	}

	var foundReverse bool
	if err := gs.Read(ctx, &spb.ReadRequest{
		Source:   edge.Target,
		EdgeKind: edges.Mirror(edge.EdgeKind),
	}, func(entry *spb.Entry) error {
		foundReverse = true
		return nil
	}); err != nil {
		return fmt.Errorf("error checking for reverse edge: %v", err)
	}
	if foundReverse {
		return nil
	}
	return addReverseEdges(ctx, gs)
}
Example #4
0
// ReverseSinglePath returns the reverse p, assuming it is a single-edge Path.
func ReverseSinglePath(p *ipb.Path) *ipb.Path {
	return &ipb.Path{
		Pivot: p.Edges[0].Target,
		Edges: []*ipb.Path_Edge{{
			Kind:    edges.Mirror(p.Edges[0].Kind),
			Ordinal: p.Edges[0].Ordinal,
			Target:  p.Pivot,
		}},
	}
}
Example #5
0
// ToString returns a human-readable string representation of p.
func ToString(p *ipb.Path) string {
	s := "**" + p.Pivot.NodeKind + "**"
	for _, e := range p.Edges {
		if edges.IsForward(e.Kind) {
			s += fmt.Sprintf(" -[%s]> %s", e.Kind, e.Target.NodeKind)
		} else {
			s += fmt.Sprintf(" <[%s]- %s", edges.Mirror(e.Kind), e.Target.NodeKind)
		}
	}
	return s
}
Example #6
0
// expandEdgeKind prefixes unrooted (not starting with "/") edge kinds with the
// standard Kythe edge prefix ("/kythe/edge/").
func expandEdgeKind(kind string) string {
	ck := edges.Canonical(kind)
	if strings.HasPrefix(ck, "/") {
		return kind
	}

	expansion := edges.Prefix + ck
	if edges.IsReverse(kind) {
		return edges.Mirror(expansion)
	}
	return expansion
}
Example #7
0
func writeCompletedEdges(ctx context.Context, output disksort.Interface, e *srvpb.Edge) error {
	if err := output.Add(&srvpb.Edge{
		Source:  &srvpb.Node{Ticket: e.Source.Ticket},
		Kind:    e.Kind,
		Ordinal: e.Ordinal,
		Target:  e.Target,
	}); err != nil {
		return fmt.Errorf("error writing complete edge: %v", err)
	}
	if err := output.Add(&srvpb.Edge{
		Source:  &srvpb.Node{Ticket: e.Target.Ticket},
		Kind:    edges.Mirror(e.Kind),
		Ordinal: e.Ordinal,
		Target:  assemble.FilterTextFacts(e.Source),
	}); err != nil {
		return fmt.Errorf("error writing complete edge mirror: %v", err)
	}
	return nil
}
Example #8
0
// PartialReverseEdges returns the set of partial reverse edges from the given source.  Each
// reversed Edge has its Target fully populated and its Source will have no facts.  To ensure every
// node has at least 1 Edge, the first Edge will be a self-edge without a Kind or Target.  To reduce
// the size of edge sets, each Target will have any text facts filtered (see FilterTextFacts).
func PartialReverseEdges(src *ipb.Source) []*srvpb.Edge {
	node := Node(src)

	result := []*srvpb.Edge{{
		Source: node, // self-edge to ensure every node has at least 1 edge
	}}

	targetNode := FilterTextFacts(node)

	for kind, group := range src.EdgeGroups {
		rev := edges.Mirror(kind)
		for _, target := range group.Edges {
			result = append(result, &srvpb.Edge{
				Source:  &srvpb.Node{Ticket: target.Ticket},
				Kind:    rev,
				Ordinal: target.Ordinal,
				Target:  targetNode,
			})
		}
	}

	return result
}
Example #9
0
// FromSource creates a set of *ipb.Paths for each of its edges as well as a
// single *ipb.Path with only its pivot set to be the source node.
func FromSource(src *ipb.Source) []*ipb.Path {
	var paths []*ipb.Path

	n := specialize(assemble.Node(src))
	paths = append(paths, &ipb.Path{Pivot: n})
	for kind, group := range src.EdgeGroups {
		if !edges.IsForward(kind) {
			continue
		}
		for _, tgt := range group.Edges {
			paths = append(paths, &ipb.Path{
				Pivot: &ipb.Path_Node{
					Ticket: tgt.Ticket,
				},
				Edges: []*ipb.Path_Edge{{
					Kind:    edges.Mirror(kind),
					Ordinal: int32(tgt.Ordinal),
					Target:  n,
				}},
			})
		}
	}
	return paths
}
Example #10
0
				Ticket: targetSet.Elements(),
				Filter: req.Filter,
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
			}
			for ticket, node := range nodesReply.Nodes {
				reply.Nodes[ticket] = node
			}
		}
	}

	return reply, nil
}

var revChildOfEdgeKind = edges.Mirror(edges.ChildOf)

func getSourceText(ctx context.Context, gs graphstore.Service, fileVName *spb.VName) (text []byte, encoding string, err error) {
	if err := gs.Read(ctx, &spb.ReadRequest{Source: fileVName}, func(entry *spb.Entry) error {
		switch entry.FactName {
		case facts.Text:
			text = entry.FactValue
		case facts.TextEncoding:
			encoding = string(entry.FactValue)
		default:
			// skip other file facts
		}
		return nil
	}); err != nil {
		return nil, "", fmt.Errorf("read error: %v", err)
	}
Example #11
0
	} `json:"span"`
	Kind string `json:"kind"`

	Node struct {
		Ticket  string   `json:"ticket"`
		Names   []string `json:"names,omitempty"`
		Kind    string   `json:"kind,omitempty"`
		Subkind string   `json:"subkind,omitempty"`
		Typed   string   `json:"typed,omitempty"`

		Definitions []*definition `json:"definitions,omitempty"`
	} `json:"node"`
}

var (
	definedAtEdge        = edges.Mirror(edges.Defines)
	definedBindingAtEdge = edges.Mirror(edges.DefinesBinding)
)

func main() {
	flag.Parse()
	if flag.NArg() > 0 {
		flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args())
	} else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) {
		flagutil.UsageError("non-negative --offset (or --line and --column) required")
	} else if *path == "" {
		flagutil.UsageError("must provide --path")
	}

	defer (*apiFlag).Close()
	xs = *apiFlag
Example #12
0
	testAnchorVName       = sig("testAnchor")
	testAnchorTargetVName = sig("someSemanticNode")

	testNodes = []*node{
		{sig("orphanedNode"), newFacts(facts.NodeKind, "orphan"), nil},
		{testFileVName, newFacts(
			facts.NodeKind, nodes.File,
			facts.Text, testFileContent,
			facts.TextEncoding, testFileEncoding), map[string][]*spb.VName{
			revChildOfEdgeKind: {testAnchorVName},
		}},
		{sig("sig2"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{
			"someEdgeKind": {sig("signature")},
		}},
		{sig("signature"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{
			edges.Mirror("someEdgeKind"): {sig("sig2")},
			edges.Param:                  {sig("sig2"), sig("someParameter")},
		}},
		{testAnchorVName, newFacts(
			facts.AnchorEnd, "4",
			facts.AnchorStart, "1",
			facts.NodeKind, nodes.Anchor,
		), map[string][]*spb.VName{
			edges.ChildOf: {testFileVName},
			edges.Ref:     {testAnchorTargetVName},
		}},
		{testAnchorTargetVName, newFacts(facts.NodeKind, "record"), map[string][]*spb.VName{
			edges.Mirror(edges.Ref): {testAnchorVName},
		}},
	}
	testEntries = nodesToEntries(testNodes)
Example #13
0
func displayEdgeGraph(reply *gpb.EdgesReply) error {
	nodes := xrefs.NodesMap(reply.Nodes)
	esets := make(map[string]map[string]stringset.Set)

	for source, es := range reply.EdgeSets {
		for gKind, g := range es.Groups {
			for _, edge := range g.Edge {
				tgt := edge.TargetTicket
				src, kind := source, gKind
				if edges.IsReverse(kind) {
					src, kind, tgt = tgt, edges.Mirror(kind), src
				}
				groups, ok := esets[src]
				if !ok {
					groups = make(map[string]stringset.Set)
					esets[src] = groups
				}
				targets, ok := groups[kind]
				if ok {
					targets.Add(tgt)
				} else {
					groups[kind] = stringset.New(tgt)
				}

			}
		}
	}
	if _, err := fmt.Println("digraph kythe {"); err != nil {
		return err
	}
	for ticket, node := range nodes {
		if _, err := fmt.Printf(`	%q [label=<<table><tr><td colspan="2">%s</td></tr>`, ticket, html.EscapeString(ticket)); err != nil {
			return err
		}
		var facts []string
		for fact := range node {
			facts = append(facts, fact)
		}
		sort.Strings(facts)
		for _, fact := range facts {
			if _, err := fmt.Printf("<tr><td>%s</td><td>%s</td></tr>", html.EscapeString(fact), html.EscapeString(string(node[fact]))); err != nil {
				return err
			}
		}
		if _, err := fmt.Println("</table>> shape=plaintext];"); err != nil {
			return err
		}
	}
	if _, err := fmt.Println(); err != nil {
		return err
	}

	for src, groups := range esets {
		for kind, targets := range groups {
			for tgt := range targets {
				if _, err := fmt.Printf("\t%q -> %q [label=%q];\n", src, tgt, kind); err != nil {
					return err
				}
			}
		}
	}
	if _, err := fmt.Println("}"); err != nil {
		return err
	}
	return nil
}
Example #14
0
func TestSlowDocumentation(t *testing.T) {
	db := []struct {
		ticket, kind, documented, defines, completes, completed, childof, typed, text, format string
		params, definitionText                                                                []string
	}{
		{ticket: "kythe://test#a", kind: "etc", documented: "kythe://test#adoc", format: "asig"},
		{ticket: "kythe://test#adoc", kind: "doc", text: "atext"},
		{ticket: "kythe://test#fdoc", kind: "doc", text: "ftext"},
		{ticket: "kythe://test#fdecl", kind: "function", documented: "kythe://test#fdoc", format: "fsig"},
		{ticket: "kythe://test#fdefn", kind: "function", completed: "kythe://test#fbind", format: "fsig", definitionText: []string{"fdeftext"}},
		{ticket: "kythe://test#fbind", kind: "anchor", defines: "kythe://test#fdefn", completes: "kythe://test#fdecl"},
		{ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc"},
		{ticket: "kythe://test#ldoc", kind: "doc", text: "ltext", params: []string{"kythe://test#l1", "kythe://test#l2"}},
		{ticket: "kythe://test#l1", kind: "etc", definitionText: []string{"deftext1"}},
		{ticket: "kythe://test#l2", kind: "etc", definitionText: []string{"deftext2"}},
		{ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc", format: "lsig"},
	}
	mkPr := func(text string, linkTicket ...string) *xpb.Printable {
		links := make([]*xpb.Link, len(linkTicket))
		for i, link := range linkTicket {
			links[i] = &xpb.Link{Definition: []string{link}}
		}
		return &xpb.Printable{RawText: text, Link: links}
	}
	type returnNode struct{ ticket, kind, anchorText string }
	tests := []struct {
		ticket string
		reply  *xpb.DocumentationReply_Document
		nodes  []returnNode
	}{
		{ticket: "kythe://test#a", reply: &xpb.DocumentationReply_Document{Signature: mkPr("asig"), Text: mkPr("atext")}, nodes: []returnNode{{ticket: "kythe://test#a", kind: "etc"}}},
		// Note that SlowDefinitions doesn't handle completions.
		{ticket: "kythe://test#fdecl", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdecl", kind: "function"}}},
		{ticket: "kythe://test#fdefn", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdefn", kind: "function", anchorText: "fdeftext"}}},
		{ticket: "kythe://test#l", reply: &xpb.DocumentationReply_Document{Signature: mkPr("lsig"), Text: mkPr("ltext", "kythe://test#l1", "kythe://test#l2")},
			nodes: []returnNode{{ticket: "kythe://test#l1", kind: "etc", anchorText: "deftext1"}, {ticket: "kythe://test#l2", kind: "etc", anchorText: "deftext2"}, {ticket: "kythe://test#l", kind: "etc"}}},
	}
	nodes := make(map[string]*cpb.NodeInfo)
	esets := make(map[string]*gpb.EdgeSet)
	xrefs := make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet)
	for _, node := range db {
		nodes[node.ticket] = &cpb.NodeInfo{
			Facts: map[string][]byte{
				facts.NodeKind: []byte(node.kind),
				facts.Text:     []byte(node.text),
				facts.Format:   []byte(node.format),
			},
		}
		if node.definitionText != nil {
			set := &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: node.ticket}
			for _, text := range node.definitionText {
				set.Definition = append(set.Definition, &xpb.CrossReferencesReply_RelatedAnchor{Anchor: &xpb.Anchor{Text: text, Ticket: node.ticket + "a"}})
			}
			xrefs[node.ticket] = set
		}
		set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
		if node.typed != "" {
			set.Groups[edges.Typed] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.typed}},
			}
		}
		if node.childof != "" {
			set.Groups[edges.ChildOf] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.childof}},
			}
		}
		if node.documented != "" {
			set.Groups[edges.Mirror(edges.Documents)] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.documented}},
			}
		}
		if node.completes != "" {
			set.Groups[edges.Completes] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completes}},
			}
		}
		if node.completed != "" {
			set.Groups[edges.Mirror(edges.Completes)] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completed}},
			}
		}
		if node.defines != "" {
			set.Groups[edges.DefinesBinding] = &gpb.EdgeSet_Group{
				Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.defines}},
			}
		}
		if node.params != nil {
			var groups []*gpb.EdgeSet_Group_Edge
			for i, p := range node.params {
				groups = append(groups, &gpb.EdgeSet_Group_Edge{TargetTicket: p, Ordinal: int32(i)})
			}
			set.Groups[edges.Param] = &gpb.EdgeSet_Group{Edge: groups}
		}
		esets[node.ticket] = set
	}
	getNode := func(ticket string, facts []string) *cpb.NodeInfo {
		data, found := nodes[ticket]
		if !found {
			return nil
		}
		info := &cpb.NodeInfo{Facts: make(map[string][]byte)}
		for _, fact := range facts {
			info.Facts[fact] = data.Facts[fact]
		}
		return info
	}
	service := &mockService{
		NodesFn: func(req *gpb.NodesRequest) (*gpb.NodesReply, error) {
			reply := &gpb.NodesReply{Nodes: make(map[string]*cpb.NodeInfo)}
			for _, ticket := range req.Ticket {
				if info := getNode(ticket, req.Filter); info != nil {
					reply.Nodes[ticket] = info
				}
			}
			return reply, nil
		},
		EdgesFn: func(req *gpb.EdgesRequest) (*gpb.EdgesReply, error) {
			reply := &gpb.EdgesReply{
				EdgeSets: make(map[string]*gpb.EdgeSet),
				Nodes:    make(map[string]*cpb.NodeInfo),
			}
			for _, ticket := range req.Ticket {
				if data, found := esets[ticket]; found {
					set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
					reply.EdgeSets[ticket] = set
					nodes := make(map[string]bool)
					for groupKind, group := range data.Groups {
						for _, kind := range req.Kind {
							if groupKind == kind {
								set.Groups[kind] = group
								for _, edge := range group.Edge {
									if !nodes[edge.TargetTicket] {
										nodes[edge.TargetTicket] = true
										if node := getNode(edge.TargetTicket, req.Filter); node != nil {
											reply.Nodes[edge.TargetTicket] = node
										}
									}
								}
								break
							}
						}
					}
				}
			}
			return reply, nil
		},
		CrossReferencesFn: func(req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
			reply := &xpb.CrossReferencesReply{
				CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),
			}
			if req.DefinitionKind != xpb.CrossReferencesRequest_BINDING_DEFINITIONS &&
				req.DefinitionKind != xpb.CrossReferencesRequest_ALL_DEFINITIONS ||
				req.ReferenceKind != xpb.CrossReferencesRequest_NO_REFERENCES ||
				req.DocumentationKind != xpb.CrossReferencesRequest_NO_DOCUMENTATION ||
				req.AnchorText {
				t.Fatalf("Unexpected CrossReferences request: %v", req)
			}
			for _, ticket := range req.Ticket {
				if set, found := xrefs[ticket]; found {
					reply.CrossReferences[ticket] = set
				}
			}
			return reply, nil
		},
	}
	for _, test := range tests {
		reply, err := SlowDocumentation(nil, service, &xpb.DocumentationRequest{Ticket: []string{test.ticket}, Filter: []string{facts.NodeKind}})
		if err != nil {
			t.Fatalf("SlowDocumentation error for %s: %v", test.ticket, err)
		}
		test.reply.Ticket = test.ticket
		expectedDoc := &xpb.DocumentationReply{Document: []*xpb.DocumentationReply_Document{test.reply}}
		for _, node := range test.nodes {
			anchorTicket := ""
			if node.anchorText != "" {
				if expectedDoc.DefinitionLocations == nil {
					expectedDoc.DefinitionLocations = make(map[string]*xpb.Anchor)
				}
				anchorTicket = node.ticket + "a"
				expectedDoc.DefinitionLocations[anchorTicket] = &xpb.Anchor{Ticket: anchorTicket, Text: node.anchorText}
			}
			if expectedDoc.Nodes == nil {
				expectedDoc.Nodes = make(map[string]*cpb.NodeInfo)
			}
			expectedDoc.Nodes[node.ticket] = &cpb.NodeInfo{Definition: anchorTicket, Facts: map[string][]byte{"/kythe/node/kind": []byte(node.kind)}}
		}
		if err := testutil.DeepEqual(expectedDoc, reply); err != nil {
			t.Fatal(err)
		}
	}
}