func addReverseEdges(ctx context.Context, gs graphstore.Service) error { log.Println("Adding reverse edges") var ( totalEntries int addedEdges int ) startTime := time.Now() err := gs.Scan(ctx, new(spb.ScanRequest), func(entry *spb.Entry) error { kind := entry.EdgeKind if kind != "" && edges.IsForward(kind) { if err := gs.Write(ctx, &spb.WriteRequest{ Source: entry.Target, Update: []*spb.WriteRequest_Update{{ Target: entry.Source, EdgeKind: edges.Mirror(kind), FactName: entry.FactName, FactValue: entry.FactValue, }}, }); err != nil { return fmt.Errorf("Failed to write reverse edge: %v", err) } addedEdges++ } totalEntries++ return nil }) log.Printf("Wrote %d reverse edges to GraphStore (%d total entries): %v", addedEdges, totalEntries, time.Since(startTime)) return err }
// CrossReference returns a (Referent, TargetAnchor) *ipb.CrossReference // equivalent to the given decoration. The decoration's anchor is expanded // given its parent file and associated Normalizer. func CrossReference(file *srvpb.File, norm *xrefs.Normalizer, d *srvpb.FileDecorations_Decoration, tgt *srvpb.Node) (*ipb.CrossReference, error) { if file == nil || norm == nil { return nil, errors.New("missing decoration's parent file") } ea, err := ExpandAnchor(d.Anchor, file, norm, edges.Mirror(d.Kind)) if err != nil { return nil, fmt.Errorf("error expanding anchor {%+v}: %v", d.Anchor, err) } // Throw away most of the referent's facts. They are not needed. var selected []*cpb.Fact if tgt != nil { for _, fact := range tgt.Fact { if fact.Name == facts.Complete { selected = append(selected, fact) } } } return &ipb.CrossReference{ Referent: &srvpb.Node{ Ticket: d.Target, Fact: selected, }, TargetAnchor: ea, }, nil }
// EnsureReverseEdges checks if gs contains reverse edges. If it doesn't, it // will scan gs for all forward edges, adding a reverse for each back into the // GraphStore. This is necessary for a GraphStoreService to work properly. func EnsureReverseEdges(ctx context.Context, gs graphstore.Service) error { var edge *spb.Entry if err := gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error { if graphstore.IsEdge(e) { edge = e return io.EOF } return nil }); err != nil { return err } if edge == nil { log.Println("No edges found in GraphStore") return nil } else if edges.IsReverse(edge.EdgeKind) { return nil } var foundReverse bool if err := gs.Read(ctx, &spb.ReadRequest{ Source: edge.Target, EdgeKind: edges.Mirror(edge.EdgeKind), }, func(entry *spb.Entry) error { foundReverse = true return nil }); err != nil { return fmt.Errorf("error checking for reverse edge: %v", err) } if foundReverse { return nil } return addReverseEdges(ctx, gs) }
// ReverseSinglePath returns the reverse p, assuming it is a single-edge Path. func ReverseSinglePath(p *ipb.Path) *ipb.Path { return &ipb.Path{ Pivot: p.Edges[0].Target, Edges: []*ipb.Path_Edge{{ Kind: edges.Mirror(p.Edges[0].Kind), Ordinal: p.Edges[0].Ordinal, Target: p.Pivot, }}, } }
// ToString returns a human-readable string representation of p. func ToString(p *ipb.Path) string { s := "**" + p.Pivot.NodeKind + "**" for _, e := range p.Edges { if edges.IsForward(e.Kind) { s += fmt.Sprintf(" -[%s]> %s", e.Kind, e.Target.NodeKind) } else { s += fmt.Sprintf(" <[%s]- %s", edges.Mirror(e.Kind), e.Target.NodeKind) } } return s }
// expandEdgeKind prefixes unrooted (not starting with "/") edge kinds with the // standard Kythe edge prefix ("/kythe/edge/"). func expandEdgeKind(kind string) string { ck := edges.Canonical(kind) if strings.HasPrefix(ck, "/") { return kind } expansion := edges.Prefix + ck if edges.IsReverse(kind) { return edges.Mirror(expansion) } return expansion }
func writeCompletedEdges(ctx context.Context, output disksort.Interface, e *srvpb.Edge) error { if err := output.Add(&srvpb.Edge{ Source: &srvpb.Node{Ticket: e.Source.Ticket}, Kind: e.Kind, Ordinal: e.Ordinal, Target: e.Target, }); err != nil { return fmt.Errorf("error writing complete edge: %v", err) } if err := output.Add(&srvpb.Edge{ Source: &srvpb.Node{Ticket: e.Target.Ticket}, Kind: edges.Mirror(e.Kind), Ordinal: e.Ordinal, Target: assemble.FilterTextFacts(e.Source), }); err != nil { return fmt.Errorf("error writing complete edge mirror: %v", err) } return nil }
// PartialReverseEdges returns the set of partial reverse edges from the given source. Each // reversed Edge has its Target fully populated and its Source will have no facts. To ensure every // node has at least 1 Edge, the first Edge will be a self-edge without a Kind or Target. To reduce // the size of edge sets, each Target will have any text facts filtered (see FilterTextFacts). func PartialReverseEdges(src *ipb.Source) []*srvpb.Edge { node := Node(src) result := []*srvpb.Edge{{ Source: node, // self-edge to ensure every node has at least 1 edge }} targetNode := FilterTextFacts(node) for kind, group := range src.EdgeGroups { rev := edges.Mirror(kind) for _, target := range group.Edges { result = append(result, &srvpb.Edge{ Source: &srvpb.Node{Ticket: target.Ticket}, Kind: rev, Ordinal: target.Ordinal, Target: targetNode, }) } } return result }
// FromSource creates a set of *ipb.Paths for each of its edges as well as a // single *ipb.Path with only its pivot set to be the source node. func FromSource(src *ipb.Source) []*ipb.Path { var paths []*ipb.Path n := specialize(assemble.Node(src)) paths = append(paths, &ipb.Path{Pivot: n}) for kind, group := range src.EdgeGroups { if !edges.IsForward(kind) { continue } for _, tgt := range group.Edges { paths = append(paths, &ipb.Path{ Pivot: &ipb.Path_Node{ Ticket: tgt.Ticket, }, Edges: []*ipb.Path_Edge{{ Kind: edges.Mirror(kind), Ordinal: int32(tgt.Ordinal), Target: n, }}, }) } } return paths }
Ticket: targetSet.Elements(), Filter: req.Filter, }) if err != nil { return nil, fmt.Errorf("failure getting reference target nodes: %v", err) } for ticket, node := range nodesReply.Nodes { reply.Nodes[ticket] = node } } } return reply, nil } var revChildOfEdgeKind = edges.Mirror(edges.ChildOf) func getSourceText(ctx context.Context, gs graphstore.Service, fileVName *spb.VName) (text []byte, encoding string, err error) { if err := gs.Read(ctx, &spb.ReadRequest{Source: fileVName}, func(entry *spb.Entry) error { switch entry.FactName { case facts.Text: text = entry.FactValue case facts.TextEncoding: encoding = string(entry.FactValue) default: // skip other file facts } return nil }); err != nil { return nil, "", fmt.Errorf("read error: %v", err) }
} `json:"span"` Kind string `json:"kind"` Node struct { Ticket string `json:"ticket"` Names []string `json:"names,omitempty"` Kind string `json:"kind,omitempty"` Subkind string `json:"subkind,omitempty"` Typed string `json:"typed,omitempty"` Definitions []*definition `json:"definitions,omitempty"` } `json:"node"` } var ( definedAtEdge = edges.Mirror(edges.Defines) definedBindingAtEdge = edges.Mirror(edges.DefinesBinding) ) func main() { flag.Parse() if flag.NArg() > 0 { flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args()) } else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) { flagutil.UsageError("non-negative --offset (or --line and --column) required") } else if *path == "" { flagutil.UsageError("must provide --path") } defer (*apiFlag).Close() xs = *apiFlag
testAnchorVName = sig("testAnchor") testAnchorTargetVName = sig("someSemanticNode") testNodes = []*node{ {sig("orphanedNode"), newFacts(facts.NodeKind, "orphan"), nil}, {testFileVName, newFacts( facts.NodeKind, nodes.File, facts.Text, testFileContent, facts.TextEncoding, testFileEncoding), map[string][]*spb.VName{ revChildOfEdgeKind: {testAnchorVName}, }}, {sig("sig2"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{ "someEdgeKind": {sig("signature")}, }}, {sig("signature"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{ edges.Mirror("someEdgeKind"): {sig("sig2")}, edges.Param: {sig("sig2"), sig("someParameter")}, }}, {testAnchorVName, newFacts( facts.AnchorEnd, "4", facts.AnchorStart, "1", facts.NodeKind, nodes.Anchor, ), map[string][]*spb.VName{ edges.ChildOf: {testFileVName}, edges.Ref: {testAnchorTargetVName}, }}, {testAnchorTargetVName, newFacts(facts.NodeKind, "record"), map[string][]*spb.VName{ edges.Mirror(edges.Ref): {testAnchorVName}, }}, } testEntries = nodesToEntries(testNodes)
func displayEdgeGraph(reply *gpb.EdgesReply) error { nodes := xrefs.NodesMap(reply.Nodes) esets := make(map[string]map[string]stringset.Set) for source, es := range reply.EdgeSets { for gKind, g := range es.Groups { for _, edge := range g.Edge { tgt := edge.TargetTicket src, kind := source, gKind if edges.IsReverse(kind) { src, kind, tgt = tgt, edges.Mirror(kind), src } groups, ok := esets[src] if !ok { groups = make(map[string]stringset.Set) esets[src] = groups } targets, ok := groups[kind] if ok { targets.Add(tgt) } else { groups[kind] = stringset.New(tgt) } } } } if _, err := fmt.Println("digraph kythe {"); err != nil { return err } for ticket, node := range nodes { if _, err := fmt.Printf(` %q [label=<<table><tr><td colspan="2">%s</td></tr>`, ticket, html.EscapeString(ticket)); err != nil { return err } var facts []string for fact := range node { facts = append(facts, fact) } sort.Strings(facts) for _, fact := range facts { if _, err := fmt.Printf("<tr><td>%s</td><td>%s</td></tr>", html.EscapeString(fact), html.EscapeString(string(node[fact]))); err != nil { return err } } if _, err := fmt.Println("</table>> shape=plaintext];"); err != nil { return err } } if _, err := fmt.Println(); err != nil { return err } for src, groups := range esets { for kind, targets := range groups { for tgt := range targets { if _, err := fmt.Printf("\t%q -> %q [label=%q];\n", src, tgt, kind); err != nil { return err } } } } if _, err := fmt.Println("}"); err != nil { return err } return nil }
func TestSlowDocumentation(t *testing.T) { db := []struct { ticket, kind, documented, defines, completes, completed, childof, typed, text, format string params, definitionText []string }{ {ticket: "kythe://test#a", kind: "etc", documented: "kythe://test#adoc", format: "asig"}, {ticket: "kythe://test#adoc", kind: "doc", text: "atext"}, {ticket: "kythe://test#fdoc", kind: "doc", text: "ftext"}, {ticket: "kythe://test#fdecl", kind: "function", documented: "kythe://test#fdoc", format: "fsig"}, {ticket: "kythe://test#fdefn", kind: "function", completed: "kythe://test#fbind", format: "fsig", definitionText: []string{"fdeftext"}}, {ticket: "kythe://test#fbind", kind: "anchor", defines: "kythe://test#fdefn", completes: "kythe://test#fdecl"}, {ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc"}, {ticket: "kythe://test#ldoc", kind: "doc", text: "ltext", params: []string{"kythe://test#l1", "kythe://test#l2"}}, {ticket: "kythe://test#l1", kind: "etc", definitionText: []string{"deftext1"}}, {ticket: "kythe://test#l2", kind: "etc", definitionText: []string{"deftext2"}}, {ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc", format: "lsig"}, } mkPr := func(text string, linkTicket ...string) *xpb.Printable { links := make([]*xpb.Link, len(linkTicket)) for i, link := range linkTicket { links[i] = &xpb.Link{Definition: []string{link}} } return &xpb.Printable{RawText: text, Link: links} } type returnNode struct{ ticket, kind, anchorText string } tests := []struct { ticket string reply *xpb.DocumentationReply_Document nodes []returnNode }{ {ticket: "kythe://test#a", reply: &xpb.DocumentationReply_Document{Signature: mkPr("asig"), Text: mkPr("atext")}, nodes: []returnNode{{ticket: "kythe://test#a", kind: "etc"}}}, // Note that SlowDefinitions doesn't handle completions. {ticket: "kythe://test#fdecl", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdecl", kind: "function"}}}, {ticket: "kythe://test#fdefn", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdefn", kind: "function", anchorText: "fdeftext"}}}, {ticket: "kythe://test#l", reply: &xpb.DocumentationReply_Document{Signature: mkPr("lsig"), Text: mkPr("ltext", "kythe://test#l1", "kythe://test#l2")}, nodes: []returnNode{{ticket: "kythe://test#l1", kind: "etc", anchorText: "deftext1"}, {ticket: "kythe://test#l2", kind: "etc", anchorText: "deftext2"}, {ticket: "kythe://test#l", kind: "etc"}}}, } nodes := make(map[string]*cpb.NodeInfo) esets := make(map[string]*gpb.EdgeSet) xrefs := make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet) for _, node := range db { nodes[node.ticket] = &cpb.NodeInfo{ Facts: map[string][]byte{ facts.NodeKind: []byte(node.kind), facts.Text: []byte(node.text), facts.Format: []byte(node.format), }, } if node.definitionText != nil { set := &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: node.ticket} for _, text := range node.definitionText { set.Definition = append(set.Definition, &xpb.CrossReferencesReply_RelatedAnchor{Anchor: &xpb.Anchor{Text: text, Ticket: node.ticket + "a"}}) } xrefs[node.ticket] = set } set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)} if node.typed != "" { set.Groups[edges.Typed] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.typed}}, } } if node.childof != "" { set.Groups[edges.ChildOf] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.childof}}, } } if node.documented != "" { set.Groups[edges.Mirror(edges.Documents)] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.documented}}, } } if node.completes != "" { set.Groups[edges.Completes] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completes}}, } } if node.completed != "" { set.Groups[edges.Mirror(edges.Completes)] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completed}}, } } if node.defines != "" { set.Groups[edges.DefinesBinding] = &gpb.EdgeSet_Group{ Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.defines}}, } } if node.params != nil { var groups []*gpb.EdgeSet_Group_Edge for i, p := range node.params { groups = append(groups, &gpb.EdgeSet_Group_Edge{TargetTicket: p, Ordinal: int32(i)}) } set.Groups[edges.Param] = &gpb.EdgeSet_Group{Edge: groups} } esets[node.ticket] = set } getNode := func(ticket string, facts []string) *cpb.NodeInfo { data, found := nodes[ticket] if !found { return nil } info := &cpb.NodeInfo{Facts: make(map[string][]byte)} for _, fact := range facts { info.Facts[fact] = data.Facts[fact] } return info } service := &mockService{ NodesFn: func(req *gpb.NodesRequest) (*gpb.NodesReply, error) { reply := &gpb.NodesReply{Nodes: make(map[string]*cpb.NodeInfo)} for _, ticket := range req.Ticket { if info := getNode(ticket, req.Filter); info != nil { reply.Nodes[ticket] = info } } return reply, nil }, EdgesFn: func(req *gpb.EdgesRequest) (*gpb.EdgesReply, error) { reply := &gpb.EdgesReply{ EdgeSets: make(map[string]*gpb.EdgeSet), Nodes: make(map[string]*cpb.NodeInfo), } for _, ticket := range req.Ticket { if data, found := esets[ticket]; found { set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)} reply.EdgeSets[ticket] = set nodes := make(map[string]bool) for groupKind, group := range data.Groups { for _, kind := range req.Kind { if groupKind == kind { set.Groups[kind] = group for _, edge := range group.Edge { if !nodes[edge.TargetTicket] { nodes[edge.TargetTicket] = true if node := getNode(edge.TargetTicket, req.Filter); node != nil { reply.Nodes[edge.TargetTicket] = node } } } break } } } } } return reply, nil }, CrossReferencesFn: func(req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) { reply := &xpb.CrossReferencesReply{ CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet), } if req.DefinitionKind != xpb.CrossReferencesRequest_BINDING_DEFINITIONS && req.DefinitionKind != xpb.CrossReferencesRequest_ALL_DEFINITIONS || req.ReferenceKind != xpb.CrossReferencesRequest_NO_REFERENCES || req.DocumentationKind != xpb.CrossReferencesRequest_NO_DOCUMENTATION || req.AnchorText { t.Fatalf("Unexpected CrossReferences request: %v", req) } for _, ticket := range req.Ticket { if set, found := xrefs[ticket]; found { reply.CrossReferences[ticket] = set } } return reply, nil }, } for _, test := range tests { reply, err := SlowDocumentation(nil, service, &xpb.DocumentationRequest{Ticket: []string{test.ticket}, Filter: []string{facts.NodeKind}}) if err != nil { t.Fatalf("SlowDocumentation error for %s: %v", test.ticket, err) } test.reply.Ticket = test.ticket expectedDoc := &xpb.DocumentationReply{Document: []*xpb.DocumentationReply_Document{test.reply}} for _, node := range test.nodes { anchorTicket := "" if node.anchorText != "" { if expectedDoc.DefinitionLocations == nil { expectedDoc.DefinitionLocations = make(map[string]*xpb.Anchor) } anchorTicket = node.ticket + "a" expectedDoc.DefinitionLocations[anchorTicket] = &xpb.Anchor{Ticket: anchorTicket, Text: node.anchorText} } if expectedDoc.Nodes == nil { expectedDoc.Nodes = make(map[string]*cpb.NodeInfo) } expectedDoc.Nodes[node.ticket] = &cpb.NodeInfo{Definition: anchorTicket, Facts: map[string][]byte{"/kythe/node/kind": []byte(node.kind)}} } if err := testutil.DeepEqual(expectedDoc, reply); err != nil { t.Fatal(err) } } }