func addReverseEdges(ctx context.Context, gs graphstore.Service) error {
log.Println("Adding reverse edges")
var (
totalEntries int
addedEdges int
)
startTime := time.Now()
err := gs.Scan(ctx, new(spb.ScanRequest), func(entry *spb.Entry) error {
kind := entry.EdgeKind
if kind != "" && edges.IsForward(kind) {
if err := gs.Write(ctx, &spb.WriteRequest{
Source: entry.Target,
Update: []*spb.WriteRequest_Update{{
Target: entry.Source,
EdgeKind: edges.Mirror(kind),
FactName: entry.FactName,
FactValue: entry.FactValue,
}},
}); err != nil {
return fmt.Errorf("Failed to write reverse edge: %v", err)
}
addedEdges++
}
totalEntries++
return nil
})
log.Printf("Wrote %d reverse edges to GraphStore (%d total entries): %v", addedEdges, totalEntries, time.Since(startTime))
return err
}
// CrossReference returns a (Referent, TargetAnchor) *ipb.CrossReference
// equivalent to the given decoration. The decoration's anchor is expanded
// given its parent file and associated Normalizer.
func CrossReference(file *srvpb.File, norm *xrefs.Normalizer, d *srvpb.FileDecorations_Decoration, tgt *srvpb.Node) (*ipb.CrossReference, error) {
if file == nil || norm == nil {
return nil, errors.New("missing decoration's parent file")
}
ea, err := ExpandAnchor(d.Anchor, file, norm, edges.Mirror(d.Kind))
if err != nil {
return nil, fmt.Errorf("error expanding anchor {%+v}: %v", d.Anchor, err)
}
// Throw away most of the referent's facts. They are not needed.
var selected []*cpb.Fact
if tgt != nil {
for _, fact := range tgt.Fact {
if fact.Name == facts.Complete {
selected = append(selected, fact)
}
}
}
return &ipb.CrossReference{
Referent: &srvpb.Node{
Ticket: d.Target,
Fact: selected,
},
TargetAnchor: ea,
}, nil
}
// EnsureReverseEdges checks if gs contains reverse edges. If it doesn't, it
// will scan gs for all forward edges, adding a reverse for each back into the
// GraphStore. This is necessary for a GraphStoreService to work properly.
func EnsureReverseEdges(ctx context.Context, gs graphstore.Service) error {
var edge *spb.Entry
if err := gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
if graphstore.IsEdge(e) {
edge = e
return io.EOF
}
return nil
}); err != nil {
return err
}
if edge == nil {
log.Println("No edges found in GraphStore")
return nil
} else if edges.IsReverse(edge.EdgeKind) {
return nil
}
var foundReverse bool
if err := gs.Read(ctx, &spb.ReadRequest{
Source: edge.Target,
EdgeKind: edges.Mirror(edge.EdgeKind),
}, func(entry *spb.Entry) error {
foundReverse = true
return nil
}); err != nil {
return fmt.Errorf("error checking for reverse edge: %v", err)
}
if foundReverse {
return nil
}
return addReverseEdges(ctx, gs)
}
// ReverseSinglePath returns the reverse p, assuming it is a single-edge Path.
func ReverseSinglePath(p *ipb.Path) *ipb.Path {
return &ipb.Path{
Pivot: p.Edges[0].Target,
Edges: []*ipb.Path_Edge{{
Kind: edges.Mirror(p.Edges[0].Kind),
Ordinal: p.Edges[0].Ordinal,
Target: p.Pivot,
}},
}
}
// ToString returns a human-readable string representation of p.
func ToString(p *ipb.Path) string {
s := "**" + p.Pivot.NodeKind + "**"
for _, e := range p.Edges {
if edges.IsForward(e.Kind) {
s += fmt.Sprintf(" -[%s]> %s", e.Kind, e.Target.NodeKind)
} else {
s += fmt.Sprintf(" <[%s]- %s", edges.Mirror(e.Kind), e.Target.NodeKind)
}
}
return s
}
// expandEdgeKind prefixes unrooted (not starting with "/") edge kinds with the
// standard Kythe edge prefix ("/kythe/edge/").
func expandEdgeKind(kind string) string {
ck := edges.Canonical(kind)
if strings.HasPrefix(ck, "/") {
return kind
}
expansion := edges.Prefix + ck
if edges.IsReverse(kind) {
return edges.Mirror(expansion)
}
return expansion
}
func writeCompletedEdges(ctx context.Context, output disksort.Interface, e *srvpb.Edge) error {
if err := output.Add(&srvpb.Edge{
Source: &srvpb.Node{Ticket: e.Source.Ticket},
Kind: e.Kind,
Ordinal: e.Ordinal,
Target: e.Target,
}); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
if err := output.Add(&srvpb.Edge{
Source: &srvpb.Node{Ticket: e.Target.Ticket},
Kind: edges.Mirror(e.Kind),
Ordinal: e.Ordinal,
Target: assemble.FilterTextFacts(e.Source),
}); err != nil {
return fmt.Errorf("error writing complete edge mirror: %v", err)
}
return nil
}
// PartialReverseEdges returns the set of partial reverse edges from the given source. Each
// reversed Edge has its Target fully populated and its Source will have no facts. To ensure every
// node has at least 1 Edge, the first Edge will be a self-edge without a Kind or Target. To reduce
// the size of edge sets, each Target will have any text facts filtered (see FilterTextFacts).
func PartialReverseEdges(src *ipb.Source) []*srvpb.Edge {
node := Node(src)
result := []*srvpb.Edge{{
Source: node, // self-edge to ensure every node has at least 1 edge
}}
targetNode := FilterTextFacts(node)
for kind, group := range src.EdgeGroups {
rev := edges.Mirror(kind)
for _, target := range group.Edges {
result = append(result, &srvpb.Edge{
Source: &srvpb.Node{Ticket: target.Ticket},
Kind: rev,
Ordinal: target.Ordinal,
Target: targetNode,
})
}
}
return result
}
// FromSource creates a set of *ipb.Paths for each of its edges as well as a
// single *ipb.Path with only its pivot set to be the source node.
func FromSource(src *ipb.Source) []*ipb.Path {
var paths []*ipb.Path
n := specialize(assemble.Node(src))
paths = append(paths, &ipb.Path{Pivot: n})
for kind, group := range src.EdgeGroups {
if !edges.IsForward(kind) {
continue
}
for _, tgt := range group.Edges {
paths = append(paths, &ipb.Path{
Pivot: &ipb.Path_Node{
Ticket: tgt.Ticket,
},
Edges: []*ipb.Path_Edge{{
Kind: edges.Mirror(kind),
Ordinal: int32(tgt.Ordinal),
Target: n,
}},
})
}
}
return paths
}
Ticket: targetSet.Elements(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
}
for ticket, node := range nodesReply.Nodes {
reply.Nodes[ticket] = node
}
}
}
return reply, nil
}
var revChildOfEdgeKind = edges.Mirror(edges.ChildOf)
func getSourceText(ctx context.Context, gs graphstore.Service, fileVName *spb.VName) (text []byte, encoding string, err error) {
if err := gs.Read(ctx, &spb.ReadRequest{Source: fileVName}, func(entry *spb.Entry) error {
switch entry.FactName {
case facts.Text:
text = entry.FactValue
case facts.TextEncoding:
encoding = string(entry.FactValue)
default:
// skip other file facts
}
return nil
}); err != nil {
return nil, "", fmt.Errorf("read error: %v", err)
}
} `json:"span"`
Kind string `json:"kind"`
Node struct {
Ticket string `json:"ticket"`
Names []string `json:"names,omitempty"`
Kind string `json:"kind,omitempty"`
Subkind string `json:"subkind,omitempty"`
Typed string `json:"typed,omitempty"`
Definitions []*definition `json:"definitions,omitempty"`
} `json:"node"`
}
var (
definedAtEdge = edges.Mirror(edges.Defines)
definedBindingAtEdge = edges.Mirror(edges.DefinesBinding)
)
func main() {
flag.Parse()
if flag.NArg() > 0 {
flagutil.UsageErrorf("unknown non-flag argument(s): %v", flag.Args())
} else if *offset < 0 && (*lineNumber < 0 || *columnOffset < 0) {
flagutil.UsageError("non-negative --offset (or --line and --column) required")
} else if *path == "" {
flagutil.UsageError("must provide --path")
}
defer (*apiFlag).Close()
xs = *apiFlag
testAnchorVName = sig("testAnchor")
testAnchorTargetVName = sig("someSemanticNode")
testNodes = []*node{
{sig("orphanedNode"), newFacts(facts.NodeKind, "orphan"), nil},
{testFileVName, newFacts(
facts.NodeKind, nodes.File,
facts.Text, testFileContent,
facts.TextEncoding, testFileEncoding), map[string][]*spb.VName{
revChildOfEdgeKind: {testAnchorVName},
}},
{sig("sig2"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{
"someEdgeKind": {sig("signature")},
}},
{sig("signature"), newFacts(facts.NodeKind, "test"), map[string][]*spb.VName{
edges.Mirror("someEdgeKind"): {sig("sig2")},
edges.Param: {sig("sig2"), sig("someParameter")},
}},
{testAnchorVName, newFacts(
facts.AnchorEnd, "4",
facts.AnchorStart, "1",
facts.NodeKind, nodes.Anchor,
), map[string][]*spb.VName{
edges.ChildOf: {testFileVName},
edges.Ref: {testAnchorTargetVName},
}},
{testAnchorTargetVName, newFacts(facts.NodeKind, "record"), map[string][]*spb.VName{
edges.Mirror(edges.Ref): {testAnchorVName},
}},
}
testEntries = nodesToEntries(testNodes)
func displayEdgeGraph(reply *gpb.EdgesReply) error {
nodes := xrefs.NodesMap(reply.Nodes)
esets := make(map[string]map[string]stringset.Set)
for source, es := range reply.EdgeSets {
for gKind, g := range es.Groups {
for _, edge := range g.Edge {
tgt := edge.TargetTicket
src, kind := source, gKind
if edges.IsReverse(kind) {
src, kind, tgt = tgt, edges.Mirror(kind), src
}
groups, ok := esets[src]
if !ok {
groups = make(map[string]stringset.Set)
esets[src] = groups
}
targets, ok := groups[kind]
if ok {
targets.Add(tgt)
} else {
groups[kind] = stringset.New(tgt)
}
}
}
}
if _, err := fmt.Println("digraph kythe {"); err != nil {
return err
}
for ticket, node := range nodes {
if _, err := fmt.Printf(` %q [label=<<table><tr><td colspan="2">%s</td></tr>`, ticket, html.EscapeString(ticket)); err != nil {
return err
}
var facts []string
for fact := range node {
facts = append(facts, fact)
}
sort.Strings(facts)
for _, fact := range facts {
if _, err := fmt.Printf("<tr><td>%s</td><td>%s</td></tr>", html.EscapeString(fact), html.EscapeString(string(node[fact]))); err != nil {
return err
}
}
if _, err := fmt.Println("</table>> shape=plaintext];"); err != nil {
return err
}
}
if _, err := fmt.Println(); err != nil {
return err
}
for src, groups := range esets {
for kind, targets := range groups {
for tgt := range targets {
if _, err := fmt.Printf("\t%q -> %q [label=%q];\n", src, tgt, kind); err != nil {
return err
}
}
}
}
if _, err := fmt.Println("}"); err != nil {
return err
}
return nil
}
func TestSlowDocumentation(t *testing.T) {
db := []struct {
ticket, kind, documented, defines, completes, completed, childof, typed, text, format string
params, definitionText []string
}{
{ticket: "kythe://test#a", kind: "etc", documented: "kythe://test#adoc", format: "asig"},
{ticket: "kythe://test#adoc", kind: "doc", text: "atext"},
{ticket: "kythe://test#fdoc", kind: "doc", text: "ftext"},
{ticket: "kythe://test#fdecl", kind: "function", documented: "kythe://test#fdoc", format: "fsig"},
{ticket: "kythe://test#fdefn", kind: "function", completed: "kythe://test#fbind", format: "fsig", definitionText: []string{"fdeftext"}},
{ticket: "kythe://test#fbind", kind: "anchor", defines: "kythe://test#fdefn", completes: "kythe://test#fdecl"},
{ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc"},
{ticket: "kythe://test#ldoc", kind: "doc", text: "ltext", params: []string{"kythe://test#l1", "kythe://test#l2"}},
{ticket: "kythe://test#l1", kind: "etc", definitionText: []string{"deftext1"}},
{ticket: "kythe://test#l2", kind: "etc", definitionText: []string{"deftext2"}},
{ticket: "kythe://test#l", kind: "etc", documented: "kythe://test#ldoc", format: "lsig"},
}
mkPr := func(text string, linkTicket ...string) *xpb.Printable {
links := make([]*xpb.Link, len(linkTicket))
for i, link := range linkTicket {
links[i] = &xpb.Link{Definition: []string{link}}
}
return &xpb.Printable{RawText: text, Link: links}
}
type returnNode struct{ ticket, kind, anchorText string }
tests := []struct {
ticket string
reply *xpb.DocumentationReply_Document
nodes []returnNode
}{
{ticket: "kythe://test#a", reply: &xpb.DocumentationReply_Document{Signature: mkPr("asig"), Text: mkPr("atext")}, nodes: []returnNode{{ticket: "kythe://test#a", kind: "etc"}}},
// Note that SlowDefinitions doesn't handle completions.
{ticket: "kythe://test#fdecl", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdecl", kind: "function"}}},
{ticket: "kythe://test#fdefn", reply: &xpb.DocumentationReply_Document{Signature: mkPr("fsig"), Text: mkPr("ftext")}, nodes: []returnNode{{ticket: "kythe://test#fdefn", kind: "function", anchorText: "fdeftext"}}},
{ticket: "kythe://test#l", reply: &xpb.DocumentationReply_Document{Signature: mkPr("lsig"), Text: mkPr("ltext", "kythe://test#l1", "kythe://test#l2")},
nodes: []returnNode{{ticket: "kythe://test#l1", kind: "etc", anchorText: "deftext1"}, {ticket: "kythe://test#l2", kind: "etc", anchorText: "deftext2"}, {ticket: "kythe://test#l", kind: "etc"}}},
}
nodes := make(map[string]*cpb.NodeInfo)
esets := make(map[string]*gpb.EdgeSet)
xrefs := make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet)
for _, node := range db {
nodes[node.ticket] = &cpb.NodeInfo{
Facts: map[string][]byte{
facts.NodeKind: []byte(node.kind),
facts.Text: []byte(node.text),
facts.Format: []byte(node.format),
},
}
if node.definitionText != nil {
set := &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: node.ticket}
for _, text := range node.definitionText {
set.Definition = append(set.Definition, &xpb.CrossReferencesReply_RelatedAnchor{Anchor: &xpb.Anchor{Text: text, Ticket: node.ticket + "a"}})
}
xrefs[node.ticket] = set
}
set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
if node.typed != "" {
set.Groups[edges.Typed] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.typed}},
}
}
if node.childof != "" {
set.Groups[edges.ChildOf] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.childof}},
}
}
if node.documented != "" {
set.Groups[edges.Mirror(edges.Documents)] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.documented}},
}
}
if node.completes != "" {
set.Groups[edges.Completes] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completes}},
}
}
if node.completed != "" {
set.Groups[edges.Mirror(edges.Completes)] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.completed}},
}
}
if node.defines != "" {
set.Groups[edges.DefinesBinding] = &gpb.EdgeSet_Group{
Edge: []*gpb.EdgeSet_Group_Edge{&gpb.EdgeSet_Group_Edge{TargetTicket: node.defines}},
}
}
if node.params != nil {
var groups []*gpb.EdgeSet_Group_Edge
for i, p := range node.params {
groups = append(groups, &gpb.EdgeSet_Group_Edge{TargetTicket: p, Ordinal: int32(i)})
}
set.Groups[edges.Param] = &gpb.EdgeSet_Group{Edge: groups}
}
esets[node.ticket] = set
}
getNode := func(ticket string, facts []string) *cpb.NodeInfo {
data, found := nodes[ticket]
if !found {
return nil
}
info := &cpb.NodeInfo{Facts: make(map[string][]byte)}
for _, fact := range facts {
info.Facts[fact] = data.Facts[fact]
}
return info
}
service := &mockService{
NodesFn: func(req *gpb.NodesRequest) (*gpb.NodesReply, error) {
reply := &gpb.NodesReply{Nodes: make(map[string]*cpb.NodeInfo)}
for _, ticket := range req.Ticket {
if info := getNode(ticket, req.Filter); info != nil {
reply.Nodes[ticket] = info
}
}
return reply, nil
},
EdgesFn: func(req *gpb.EdgesRequest) (*gpb.EdgesReply, error) {
reply := &gpb.EdgesReply{
EdgeSets: make(map[string]*gpb.EdgeSet),
Nodes: make(map[string]*cpb.NodeInfo),
}
for _, ticket := range req.Ticket {
if data, found := esets[ticket]; found {
set := &gpb.EdgeSet{Groups: make(map[string]*gpb.EdgeSet_Group)}
reply.EdgeSets[ticket] = set
nodes := make(map[string]bool)
for groupKind, group := range data.Groups {
for _, kind := range req.Kind {
if groupKind == kind {
set.Groups[kind] = group
for _, edge := range group.Edge {
if !nodes[edge.TargetTicket] {
nodes[edge.TargetTicket] = true
if node := getNode(edge.TargetTicket, req.Filter); node != nil {
reply.Nodes[edge.TargetTicket] = node
}
}
}
break
}
}
}
}
}
return reply, nil
},
CrossReferencesFn: func(req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
reply := &xpb.CrossReferencesReply{
CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),
}
if req.DefinitionKind != xpb.CrossReferencesRequest_BINDING_DEFINITIONS &&
req.DefinitionKind != xpb.CrossReferencesRequest_ALL_DEFINITIONS ||
req.ReferenceKind != xpb.CrossReferencesRequest_NO_REFERENCES ||
req.DocumentationKind != xpb.CrossReferencesRequest_NO_DOCUMENTATION ||
req.AnchorText {
t.Fatalf("Unexpected CrossReferences request: %v", req)
}
for _, ticket := range req.Ticket {
if set, found := xrefs[ticket]; found {
reply.CrossReferences[ticket] = set
}
}
return reply, nil
},
}
for _, test := range tests {
reply, err := SlowDocumentation(nil, service, &xpb.DocumentationRequest{Ticket: []string{test.ticket}, Filter: []string{facts.NodeKind}})
if err != nil {
t.Fatalf("SlowDocumentation error for %s: %v", test.ticket, err)
}
test.reply.Ticket = test.ticket
expectedDoc := &xpb.DocumentationReply{Document: []*xpb.DocumentationReply_Document{test.reply}}
for _, node := range test.nodes {
anchorTicket := ""
if node.anchorText != "" {
if expectedDoc.DefinitionLocations == nil {
expectedDoc.DefinitionLocations = make(map[string]*xpb.Anchor)
}
anchorTicket = node.ticket + "a"
expectedDoc.DefinitionLocations[anchorTicket] = &xpb.Anchor{Ticket: anchorTicket, Text: node.anchorText}
}
if expectedDoc.Nodes == nil {
expectedDoc.Nodes = make(map[string]*cpb.NodeInfo)
}
expectedDoc.Nodes[node.ticket] = &cpb.NodeInfo{Definition: anchorTicket, Facts: map[string][]byte{"/kythe/node/kind": []byte(node.kind)}}
}
if err := testutil.DeepEqual(expectedDoc, reply); err != nil {
t.Fatal(err)
}
}
}