// SourceFromEntries returns a new Source from the given a set of entries with // the same source VName. func SourceFromEntries(entries []*spb.Entry) *Source { if len(entries) == 0 { return nil } src := &Source{ Ticket: kytheuri.ToString(entries[0].Source), Facts: make(map[string][]byte), Edges: make(map[string][]string), } edgeTargets := make(map[string]stringset.Set) for _, e := range entries { if graphstore.IsEdge(e) { tgts, ok := edgeTargets[e.EdgeKind] if !ok { tgts = stringset.New() edgeTargets[e.EdgeKind] = tgts } tgts.Add(kytheuri.ToString(e.Target)) } else { src.Facts[e.FactName] = e.FactValue } } for kind, targets := range edgeTargets { src.Edges[kind] = targets.Slice() sort.Strings(src.Edges[kind]) } return src }
func TestDecorations(t *testing.T) { xs := newService(t, testEntries) reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{ Location: &xpb.Location{ Ticket: kytheuri.ToString(testFileVName), }, SourceText: true, References: true, Filter: []string{"**"}, }) if err != nil { t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err) } if string(reply.SourceText) != testFileContent { t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent) } if reply.Encoding != testFileEncoding { t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding) } expectedRefs := []*xpb.DecorationsReply_Reference{ { SourceTicket: kytheuri.ToString(testAnchorVName), TargetTicket: kytheuri.ToString(testAnchorTargetVName), Kind: schema.RefEdge, AnchorStart: &xpb.Location_Point{ ByteOffset: 1, LineNumber: 1, ColumnOffset: 1, }, AnchorEnd: &xpb.Location_Point{ ByteOffset: 4, LineNumber: 1, ColumnOffset: 4, }, }, } if err := testutil.DeepEqual(sortRefs(expectedRefs), sortRefs(reply.Reference)); err != nil { t.Error(err) } expectedNodes := nodesToInfos(testNodes[4:6]) if err := testutil.DeepEqual(expectedNodes, sortInfos(reply.Node)); err != nil { t.Error(err) } }
func nodesToTickets(nodes []*node) []string { var tickets []string for _, n := range nodes { tickets = append(tickets, kytheuri.ToString(n.Source)) } return tickets }
func nodesToInfos(nodes []*node) map[string]*cpb.NodeInfo { m := make(map[string]*cpb.NodeInfo) for _, n := range nodes { m[kytheuri.ToString(n.Source)] = n.Info() } return m }
func TestForPackage(t *testing.T) { tests := []struct { path string ticket string isRoot bool }{ {path: "bytes", ticket: "kythe://golang.org?lang=go?path=bytes#%3Apkg%3A", isRoot: true}, {path: "go/types", ticket: "kythe://golang.org?lang=go?path=go/types#%3Apkg%3A", isRoot: true}, {path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"}, {path: "code.google.com/p/foo.bar/baz", ticket: "kythe://code.google.com/p/foo?lang=go?path=baz?root=bar#%3Apkg%3A"}, {path: "fuzzy1.googlecode.com/alpha", ticket: "kythe://fuzzy1.googlecode.com?lang=go?path=alpha#%3Apkg%3A"}, {path: "github.com/google/kythe/foo", ticket: "kythe://github.com/google/kythe?lang=go?path=foo#%3Apkg%3A"}, {path: "bitbucket.org/zut/alors/non", ticket: "kythe://bitbucket.org/zut/alors?lang=go?path=non#%3Apkg%3A"}, {path: "launchpad.net/~frood/blee/blor", ticket: "kythe://launchpad.net/~frood/blee?lang=go?path=blor#%3Apkg%3A"}, {path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"}, } for _, test := range tests { pkg := &build.Package{ ImportPath: test.path, Goroot: test.isRoot, } got := ForPackage("", pkg) gotTicket := kytheuri.ToString(got) if gotTicket != test.ticket { t.Errorf(`ForPackage("", [%s]): got %q, want %q`, test.path, gotTicket, test.ticket) } } }
func collectNodes(nodeEntries <-chan *spb.Entry) <-chan *srvpb.Node { nodes := make(chan *srvpb.Node) go func() { var ( node *srvpb.Node vname *spb.VName ) for e := range nodeEntries { if node != nil && !compare.VNamesEqual(e.Source, vname) { nodes <- node node = nil vname = nil } if node == nil { vname = e.Source ticket := kytheuri.ToString(vname) node = &srvpb.Node{Ticket: ticket} } node.Fact = append(node.Fact, &srvpb.Node_Fact{ Name: e.FactName, Value: e.FactValue, }) } if node != nil { nodes <- node } close(nodes) }() return nodes }
// Sources constructs a new Source for every contiguous set of entries sharing // the same Source, calling f for each. func Sources(rd stream.EntryReader, f func(*ipb.Source) error) error { var source *spb.VName var src *ipb.Source if err := rd(func(entry *spb.Entry) error { if src != nil && !compare.VNamesEqual(source, entry.Source) { if err := f(src); err != nil { return err } src = nil } if src == nil { source = entry.Source src = &ipb.Source{ Ticket: kytheuri.ToString(entry.Source), Facts: make(map[string][]byte), EdgeGroups: make(map[string]*ipb.Source_EdgeGroup), } } AppendEntry(src, entry) return nil }); err != nil { return err } if src != nil { return f(src) } return nil }
// AppendEntry adds the given Entry to the Source's facts or edges. It is // assumed that src.Ticket == kytheuri.ToString(e.Source). func AppendEntry(src *ipb.Source, e *spb.Entry) { if graphstore.IsEdge(e) { kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind) group, ok := src.EdgeGroups[kind] if !ok { group = &ipb.Source_EdgeGroup{} src.EdgeGroups[kind] = group } ticket := kytheuri.ToString(e.Target) ord := int32(ordinal) for _, edge := range group.Edges { if edge.Ticket == ticket && edge.Ordinal == ord { // Don't add duplicate edge return } } group.Edges = append(group.Edges, &ipb.Source_Edge{ Ticket: ticket, Ordinal: ord, }) } else { src.Facts[e.FactName] = e.FactValue } }
func (n *node) EdgeSet() *xpb.EdgeSet { var groups []*xpb.EdgeSet_Group for kind, targets := range n.Edges { var tickets []string for _, target := range targets { tickets = append(tickets, kytheuri.ToString(target)) } groups = append(groups, &xpb.EdgeSet_Group{ Kind: kind, TargetTicket: tickets, }) } return &xpb.EdgeSet{ SourceTicket: kytheuri.ToString(n.Source), Group: groups, } }
func writeEdges(ctx context.Context, t table.Proto, edges <-chan *spb.Entry, maxEdgePageSize int) error { defer drainEntries(edges) // ensure channel is drained on errors temp, err := tempTable("edge.groups") if err != nil { return fmt.Errorf("failed to create temporary table: %v", err) } edgeGroups := &table.KVProto{temp} defer func() { if err := edgeGroups.Close(ctx); err != nil { log.Println("Error closing edge groups table: %v", err) } }() log.Println("Writing temporary edges table") var ( src *spb.VName kind string targets stringset.Set ) for e := range edges { if src != nil && (!compare.VNamesEqual(e.Source, src) || kind != e.EdgeKind) { if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil { return err } src = nil } if src == nil { src = e.Source kind = e.EdgeKind targets = stringset.New() } targets.Add(kytheuri.ToString(e.Target)) } if src != nil { if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil { return err } } return writeEdgePages(ctx, t, edgeGroups, maxEdgePageSize) }
func nodesToEdgeSets(nodes []*node) map[string]*gpb.EdgeSet { sets := make(map[string]*gpb.EdgeSet) for _, n := range nodes { set := n.EdgeSet() if len(set.Groups) > 0 { sets[kytheuri.ToString(n.Source)] = set } } return sets }
func (n *node) Info() *xpb.NodeInfo { info := &xpb.NodeInfo{ Ticket: kytheuri.ToString(n.Source), } for name, val := range n.Facts { info.Fact = append(info.Fact, &xpb.Fact{ Name: name, Value: []byte(val), }) } return info }
func node(v *spb.VName, facts ...string) *srvpb.Node { if len(facts)%2 != 0 { panic("odd number of facts") } n := &srvpb.Node{Ticket: kytheuri.ToString(v)} for i := 0; i < len(facts); i += 2 { n.Fact = append(n.Fact, &srvpb.Node_Fact{ Name: facts[i], Value: []byte(facts[i+1]), }) } return n }
func TestDecorations(t *testing.T) { xs := newService(t, testEntries) reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{ Location: &xpb.Location{ Ticket: kytheuri.ToString(testFileVName), }, SourceText: true, References: true, }) if err != nil { t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err) } if string(reply.SourceText) != testFileContent { t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent) } if reply.Encoding != testFileEncoding { t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding) } expectedRefs := []*xpb.DecorationsReply_Reference{ { SourceTicket: kytheuri.ToString(testAnchorVName), TargetTicket: kytheuri.ToString(testAnchorTargetVName), Kind: schema.RefEdge, }, } if !reflect.DeepEqual(sortRefs(reply.Reference), sortRefs(expectedRefs)) { t.Errorf("Got %v; Expected references %v", reply.Reference, expectedRefs) } refNodes := testNodes[4:6] expectedNodes := nodesToInfos(refNodes) if !reflect.DeepEqual(sortInfos(reply.Node), expectedNodes) { t.Errorf("Got %v; Expected nodes %v", reply.Node, expectedNodes) } }
func (n *node) EdgeSet() *gpb.EdgeSet { groups := make(map[string]*gpb.EdgeSet_Group) for kind, targets := range n.Edges { var edges []*gpb.EdgeSet_Group_Edge for ordinal, target := range targets { edges = append(edges, &gpb.EdgeSet_Group_Edge{ TargetTicket: kytheuri.ToString(target), Ordinal: int32(ordinal), }) } groups[kind] = &gpb.EdgeSet_Group{ Edge: edges, } } return &gpb.EdgeSet{ Groups: groups, } }
// SourceFromEntries returns a new Source from the given a set of entries with // the same source VName. func SourceFromEntries(entries []*spb.Entry) *ipb.Source { if len(entries) == 0 { return nil } src := &ipb.Source{ Ticket: kytheuri.ToString(entries[0].Source), Facts: make(map[string][]byte), EdgeGroups: make(map[string]*ipb.Source_EdgeGroup), } for _, e := range entries { AppendEntry(src, e) } for _, group := range src.EdgeGroups { sort.Sort(byOrdinal(group.Edges)) } return src }
// AddFile adds the given file VName to m. func (m *Map) AddFile(file *spb.VName) { ticket := kytheuri.ToString(file) dirPath := CleanDirPath(path.Dir(file.Path)) dir := m.ensureDir(file.Corpus, file.Root, dirPath) dir.File = addToSet(dir.File, ticket) }
// Run writes the xrefs and filetree serving tables to db based on the given // graphstore.Service. func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB) error { log.Println("Starting serving pipeline") tbl := &table.KVProto{db} // TODO(schroederc): for large corpora, this won't fit in memory var files []string entries := make(chan *spb.Entry) ftIn, nIn, eIn := make(chan *spb.VName), make(chan *spb.Entry), make(chan *spb.Entry) go func() { for entry := range entries { if entry.EdgeKind == "" { nIn <- entry if entry.FactName == schema.NodeKindFact && string(entry.FactValue) == "file" { ftIn <- entry.Source files = append(files, kytheuri.ToString(entry.Source)) } } else { eIn <- entry } } close(ftIn) close(nIn) close(eIn) }() log.Println("Scanning GraphStore") var sErr error go func() { sErr = gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error { entries <- e return nil }) close(entries) }() var ( ftErr, nErr, eErr error ftWG, edgeNodeWG sync.WaitGroup ) ftWG.Add(1) go func() { defer ftWG.Done() ftErr = writeFileTree(ctx, tbl, ftIn) log.Println("Wrote FileTree") }() edgeNodeWG.Add(2) nodes := make(chan *srvpb.Node) go func() { defer edgeNodeWG.Done() nErr = writeNodes(tbl, nIn, nodes) log.Println("Wrote Nodes") }() go func() { defer edgeNodeWG.Done() eErr = writeEdges(ctx, tbl, eIn) log.Println("Wrote Edges") }() var ( idxWG sync.WaitGroup idxErr error ) idxWG.Add(1) go func() { defer idxWG.Done() idxErr = writeIndex(&table.KVInverted{db}, nodes) log.Println("Wrote Search Index") }() edgeNodeWG.Wait() if eErr != nil { return eErr } else if nErr != nil { return nErr } es := xrefs.NodesEdgesService(&xsrv.Table{tbl}) if err := writeDecorations(ctx, tbl, es, files); err != nil { return err } ftWG.Wait() if ftErr != nil { return ftErr } idxWG.Wait() if idxErr != nil { return idxErr } return sErr }
func writeEdgePages(ctx context.Context, t table.Proto, gs graphstore.Service) error { // TODO(schroederc): spill large PagedEdgeSets into EdgePages log.Println("Writing EdgeSets") var ( lastSrc *spb.VName pes *srvpb.PagedEdgeSet grp *srvpb.EdgeSet_Group pesTotal int ) if err := gs.Scan(ctx, new(spb.ScanRequest), func(e *spb.Entry) error { if e.EdgeKind == "" { panic("non-edge entry") } if pes != nil && !compare.VNamesEqual(lastSrc, e.Source) { if grp != nil { pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp) pesTotal += len(grp.TargetTicket) } pes.TotalEdges = int32(pesTotal) if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil { return err } pes = nil grp = nil pesTotal = 0 } if pes == nil { pes = &srvpb.PagedEdgeSet{ EdgeSet: &srvpb.EdgeSet{ SourceTicket: kytheuri.ToString(e.Source), }, } } if grp != nil && grp.Kind != e.EdgeKind { pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp) pesTotal += len(grp.TargetTicket) grp = nil } if grp == nil { grp = &srvpb.EdgeSet_Group{ Kind: e.EdgeKind, } } grp.TargetTicket = append(grp.TargetTicket, kytheuri.ToString(e.Target)) lastSrc = e.Source return nil }); err != nil { return err } if pes != nil { if grp != nil { pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp) pesTotal += len(grp.TargetTicket) } pes.TotalEdges = int32(pesTotal) if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil { return err } } return nil }
// Edges implements part of the Service interface. func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) { if len(req.Ticket) == 0 { return nil, errors.New("no tickets specified") } else if req.PageToken != "" { return nil, errors.New("UNIMPLEMENTED: page_token") } patterns := xrefs.ConvertFilters(req.Filter) allowedKinds := stringset.New(req.Kind...) targetSet := stringset.New() reply := new(xpb.EdgesReply) for _, ticket := range req.Ticket { vname, err := kytheuri.ToVName(ticket) if err != nil { return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err) } var ( // EdgeKind -> StringSet<TargetTicket> filteredEdges = make(map[string]stringset.Set) filteredFacts []*xpb.Fact ) if err := g.gs.Read(ctx, &spb.ReadRequest{ Source: vname, EdgeKind: "*", }, func(entry *spb.Entry) error { edgeKind := entry.EdgeKind if edgeKind == "" { // node fact if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) { filteredFacts = append(filteredFacts, entryToFact(entry)) } } else { // edge if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) { targets := filteredEdges[edgeKind] if targets == nil { targets = stringset.New() filteredEdges[edgeKind] = targets } targets.Add(kytheuri.ToString(entry.Target)) } } return nil }); err != nil { return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket) } // Only add a EdgeSet if there are targets for the requested edge kinds. if len(filteredEdges) > 0 { var groups []*xpb.EdgeSet_Group for edgeKind, targets := range filteredEdges { g := &xpb.EdgeSet_Group{Kind: edgeKind} for target := range targets { g.TargetTicket = append(g.TargetTicket, target) targetSet.Add(target) } groups = append(groups, g) } reply.EdgeSet = append(reply.EdgeSet, &xpb.EdgeSet{ SourceTicket: ticket, Group: groups, }) // In addition, only add a NodeInfo if the filters have resulting facts. if len(filteredFacts) > 0 { reply.Node = append(reply.Node, &xpb.NodeInfo{ Ticket: ticket, Fact: filteredFacts, }) } } } // Only request Nodes when there are fact filters given. if len(req.Filter) > 0 { // Ensure reply.Node is a unique set by removing already requested nodes from targetSet for _, n := range reply.Node { targetSet.Remove(n.Ticket) } // Batch request all leftover target nodes nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{ Ticket: targetSet.Slice(), Filter: req.Filter, }) if err != nil { return nil, fmt.Errorf("failure getting target nodes: %v", err) } reply.Node = append(reply.Node, nodesReply.Node...) } return reply, nil }
func (d *DB) copyEntries(entries <-chan *spb.Entry) error { // Start a transaction for a COPY statement per table nodesTx, err := d.Begin() if err != nil { return err } edgesTx, err := d.Begin() if err != nil { return err } // Create each table in their corresponding transactions to speed up COPY if _, err := nodesTx.Exec(createNodesTable); err != nil { return fmt.Errorf("error truncating Nodes table: %v", err) } else if _, err := edgesTx.Exec(createEdgeTable); err != nil { return fmt.Errorf("error truncating Edges table: %v", err) } copyNode, err := nodesTx.Prepare(pq.CopyIn( "nodes", "ticket", "node_kind", "subkind", "text", "text_encoding", "start_offset", "end_offset", "snippet_start", "snippet_end", "other_facts_num", "other_facts", )) if err != nil { return fmt.Errorf("error preparing Nodes copy: %v", err) } copyEdge, err := edgesTx.Prepare(pq.CopyIn( "edges", "source", "kind", "target", "ordinal", )) if err != nil { return fmt.Errorf("error preparing Edges copy: %v", err) } var node srvpb.Node var nodeKind string var subkind, textEncoding *string var text *[]byte var startOffset, endOffset, snippetStart, snippetEnd *int64 for e := range entries { if graphstore.IsNodeFact(e) { ticket := kytheuri.ToString(e.Source) if node.Ticket != "" && node.Ticket != ticket { nodeTicket := node.Ticket node.Ticket = "" var rec []byte if len(node.Fact) > 0 { rec, err = proto.Marshal(&node) if err != nil { return fmt.Errorf("error marshaling facts: %v", err) } } if text != nil && textEncoding == nil { textEncoding = proto.String(facts.DefaultTextEncoding) } if _, err := copyNode.Exec( nodeTicket, nodeKind, subkind, text, textEncoding, startOffset, endOffset, snippetStart, snippetEnd, len(node.Fact), rec, ); err != nil { return fmt.Errorf("error copying node: %v", err) } node.Fact, text = node.Fact[0:0], nil nodeKind = "" subkind, textEncoding = nil, nil startOffset, endOffset, snippetStart, snippetEnd = nil, nil, nil, nil } if node.Ticket == "" { node.Ticket = ticket } switch e.FactName { case facts.NodeKind: nodeKind = string(e.FactValue) case facts.Subkind: subkind = proto.String(string(e.FactValue)) case facts.Text: text = &e.FactValue case facts.TextEncoding: textEncoding = proto.String(string(e.FactValue)) case facts.AnchorStart: n, err := strconv.ParseInt(string(e.FactValue), 10, 64) if err == nil { startOffset = proto.Int64(n) } case facts.AnchorEnd: n, err := strconv.ParseInt(string(e.FactValue), 10, 64) if err == nil { endOffset = proto.Int64(n) } case facts.SnippetStart: n, err := strconv.ParseInt(string(e.FactValue), 10, 64) if err == nil { snippetStart = proto.Int64(n) } case facts.SnippetEnd: n, err := strconv.ParseInt(string(e.FactValue), 10, 64) if err == nil { snippetEnd = proto.Int64(n) } default: node.Fact = append(node.Fact, &cpb.Fact{ Name: e.FactName, Value: e.FactValue, }) } } else if edges.IsForward(e.EdgeKind) { kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind) ticket := kytheuri.ToString(e.Source) if _, err := copyEdge.Exec(ticket, kind, kytheuri.ToString(e.Target), ordinal); err != nil { return fmt.Errorf("error copying edge: %v", err) } } } if _, err := copyNode.Exec(); err != nil { return fmt.Errorf("error flushing nodes: %v", err) } else if _, err := copyEdge.Exec(); err != nil { return fmt.Errorf("error flushing edges: %v", err) } if err := nodesTx.Commit(); err != nil { return fmt.Errorf("error committing Nodes transaction: %v", err) } else if err := edgesTx.Commit(); err != nil { return fmt.Errorf("error committing Edges transaction: %v", err) } return nil }
// Edges implements part of the Service interface. func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) { if len(req.Ticket) == 0 { return nil, errors.New("no tickets specified") } else if req.PageToken != "" { return nil, errors.New("UNIMPLEMENTED: page_token") } patterns := xrefs.ConvertFilters(req.Filter) allowedKinds := stringset.New(req.Kind...) targetSet := stringset.New() reply := &xpb.EdgesReply{ EdgeSets: make(map[string]*xpb.EdgeSet), Nodes: make(map[string]*xpb.NodeInfo), } for _, ticket := range req.Ticket { vname, err := kytheuri.ToVName(ticket) if err != nil { return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err) } var ( // EdgeKind -> TargetTicket -> OrdinalSet filteredEdges = make(map[string]map[string]map[int32]struct{}) filteredFacts = make(map[string][]byte) ) if err := g.gs.Read(ctx, &spb.ReadRequest{ Source: vname, EdgeKind: "*", }, func(entry *spb.Entry) error { edgeKind := entry.EdgeKind if edgeKind == "" { // node fact if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) { filteredFacts[entry.FactName] = entry.FactValue } } else { // edge edgeKind, ordinal, _ := schema.ParseOrdinal(edgeKind) if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) { targets, ok := filteredEdges[edgeKind] if !ok { targets = make(map[string]map[int32]struct{}) filteredEdges[edgeKind] = targets } ticket := kytheuri.ToString(entry.Target) ordSet, ok := targets[ticket] if !ok { ordSet = make(map[int32]struct{}) targets[ticket] = ordSet } ordSet[int32(ordinal)] = struct{}{} } } return nil }); err != nil { return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket) } // Only add a EdgeSet if there are targets for the requested edge kinds. if len(filteredEdges) > 0 { groups := make(map[string]*xpb.EdgeSet_Group) for edgeKind, targets := range filteredEdges { g := &xpb.EdgeSet_Group{} for target, ordinals := range targets { for ordinal := range ordinals { g.Edge = append(g.Edge, &xpb.EdgeSet_Group_Edge{ TargetTicket: target, Ordinal: ordinal, }) } targetSet.Add(target) } groups[edgeKind] = g } reply.EdgeSets[ticket] = &xpb.EdgeSet{ Groups: groups, } // In addition, only add a NodeInfo if the filters have resulting facts. if len(filteredFacts) > 0 { reply.Nodes[ticket] = &xpb.NodeInfo{ Facts: filteredFacts, } } } } // Only request Nodes when there are fact filters given. if len(req.Filter) > 0 { // Eliminate redundant work by removing already requested nodes from targetSet for ticket := range reply.Nodes { targetSet.Remove(ticket) } // Batch request all leftover target nodes nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{ Ticket: targetSet.Slice(), Filter: req.Filter, }) if err != nil { return nil, fmt.Errorf("failure getting target nodes: %v", err) } for ticket, node := range nodesReply.Nodes { reply.Nodes[ticket] = node } } return reply, nil }
// Decorations implements part of the Service interface. func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) { if len(req.DirtyBuffer) > 0 { return nil, errors.New("UNIMPLEMENTED: dirty buffers") } else if req.GetLocation() == nil { // TODO(schroederc): allow empty location when given dirty buffer return nil, errors.New("missing location") } fileVName, err := kytheuri.ToVName(req.Location.Ticket) if err != nil { return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err) } text, encoding, err := getSourceText(ctx, g.gs, fileVName) if err != nil { return nil, fmt.Errorf("failed to retrieve file text: %v", err) } norm := xrefs.NewNormalizer(text) loc, err := norm.Location(req.GetLocation()) if err != nil { return nil, err } reply := &xpb.DecorationsReply{ Location: loc, Nodes: make(map[string]*xpb.NodeInfo), } // Handle DecorationsRequest.SourceText switch if req.SourceText { if loc.Kind == xpb.Location_FILE { reply.SourceText = text } else { reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset] } reply.Encoding = encoding } // Handle DecorationsRequest.References switch if req.References { // Traverse the following chain of edges: // file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target // // Add []anchor and []target nodes to reply.Nodes // Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference patterns := xrefs.ConvertFilters(req.Filter) children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool { return e.EdgeKind == revChildOfEdgeKind }) if err != nil { return nil, fmt.Errorf("failed to retrieve file children: %v", err) } targetSet := stringset.New() for _, edge := range children { anchor := edge.Target ticket := kytheuri.ToString(anchor) anchorNodeReply, err := g.Nodes(ctx, &xpb.NodesRequest{ Ticket: []string{ticket}, }) if err != nil { return nil, fmt.Errorf("failure getting reference source node: %v", err) } else if len(anchorNodeReply.Nodes) != 1 { return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor) } node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket] if !ok { return nil, fmt.Errorf("failed to find info for node %q", ticket) } else if string(node[schema.NodeKindFact]) != schema.AnchorKind { // Skip child if it isn't an anchor node continue } anchorStart, err := strconv.Atoi(string(node[schema.AnchorStartFact])) if err != nil { log.Printf("Invalid anchor start offset %q for node %q: %v", node[schema.AnchorStartFact], ticket, err) continue } anchorEnd, err := strconv.Atoi(string(node[schema.AnchorEndFact])) if err != nil { log.Printf("Invalid anchor end offset %q for node %q: %v", node[schema.AnchorEndFact], ticket, err) continue } if loc.Kind == xpb.Location_SPAN { // Check if anchor fits within/around requested source text window if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) { continue } else if anchorStart > anchorEnd { log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd) continue } } targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool { return schema.EdgeDirection(e.EdgeKind) == schema.Forward && e.EdgeKind != schema.ChildOfEdge }) if err != nil { return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err) } if len(targets) == 0 { log.Printf("Anchor missing forward edges: {%+v}", anchor) continue } if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil { reply.Nodes[ticket] = node } for _, edge := range targets { targetTicket := kytheuri.ToString(edge.Target) targetSet.Add(targetTicket) reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{ SourceTicket: ticket, Kind: edge.Kind, TargetTicket: targetTicket, AnchorStart: norm.ByteOffset(int32(anchorStart)), AnchorEnd: norm.ByteOffset(int32(anchorEnd)), }) } } sort.Sort(bySpan(reply.Reference)) // Only request Nodes when there are fact filters given. if len(req.Filter) > 0 { // Ensure returned nodes are not duplicated. for ticket := range reply.Nodes { targetSet.Remove(ticket) } // Batch request all Reference target nodes nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{ Ticket: targetSet.Slice(), Filter: req.Filter, }) if err != nil { return nil, fmt.Errorf("failure getting reference target nodes: %v", err) } for ticket, node := range nodesReply.Nodes { reply.Nodes[ticket] = node } } } return reply, nil }
// AddFile adds the given file VName to m. func (m *Map) AddFile(file *spb.VName) { ticket := kytheuri.ToString(file) path := filepath.Join("/", file.Path) dir := m.ensureDir(file.Corpus, file.Root, filepath.Dir(path)) dir.File = addToSet(dir.File, ticket) }