// SourceFromEntries returns a new Source from the given a set of entries with
// the same source VName.
func SourceFromEntries(entries []*spb.Entry) *Source {
if len(entries) == 0 {
return nil
}
src := &Source{
Ticket: kytheuri.ToString(entries[0].Source),
Facts: make(map[string][]byte),
Edges: make(map[string][]string),
}
edgeTargets := make(map[string]stringset.Set)
for _, e := range entries {
if graphstore.IsEdge(e) {
tgts, ok := edgeTargets[e.EdgeKind]
if !ok {
tgts = stringset.New()
edgeTargets[e.EdgeKind] = tgts
}
tgts.Add(kytheuri.ToString(e.Target))
} else {
src.Facts[e.FactName] = e.FactValue
}
}
for kind, targets := range edgeTargets {
src.Edges[kind] = targets.Slice()
sort.Strings(src.Edges[kind])
}
return src
}
func TestDecorations(t *testing.T) {
xs := newService(t, testEntries)
reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
Location: &xpb.Location{
Ticket: kytheuri.ToString(testFileVName),
},
SourceText: true,
References: true,
Filter: []string{"**"},
})
if err != nil {
t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err)
}
if string(reply.SourceText) != testFileContent {
t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent)
}
if reply.Encoding != testFileEncoding {
t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding)
}
expectedRefs := []*xpb.DecorationsReply_Reference{
{
SourceTicket: kytheuri.ToString(testAnchorVName),
TargetTicket: kytheuri.ToString(testAnchorTargetVName),
Kind: schema.RefEdge,
AnchorStart: &xpb.Location_Point{
ByteOffset: 1,
LineNumber: 1,
ColumnOffset: 1,
},
AnchorEnd: &xpb.Location_Point{
ByteOffset: 4,
LineNumber: 1,
ColumnOffset: 4,
},
},
}
if err := testutil.DeepEqual(sortRefs(expectedRefs), sortRefs(reply.Reference)); err != nil {
t.Error(err)
}
expectedNodes := nodesToInfos(testNodes[4:6])
if err := testutil.DeepEqual(expectedNodes, sortInfos(reply.Node)); err != nil {
t.Error(err)
}
}
func nodesToTickets(nodes []*node) []string {
var tickets []string
for _, n := range nodes {
tickets = append(tickets, kytheuri.ToString(n.Source))
}
return tickets
}
func nodesToInfos(nodes []*node) map[string]*cpb.NodeInfo {
m := make(map[string]*cpb.NodeInfo)
for _, n := range nodes {
m[kytheuri.ToString(n.Source)] = n.Info()
}
return m
}
func TestForPackage(t *testing.T) {
tests := []struct {
path string
ticket string
isRoot bool
}{
{path: "bytes", ticket: "kythe://golang.org?lang=go?path=bytes#%3Apkg%3A", isRoot: true},
{path: "go/types", ticket: "kythe://golang.org?lang=go?path=go/types#%3Apkg%3A", isRoot: true},
{path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"},
{path: "code.google.com/p/foo.bar/baz", ticket: "kythe://code.google.com/p/foo?lang=go?path=baz?root=bar#%3Apkg%3A"},
{path: "fuzzy1.googlecode.com/alpha", ticket: "kythe://fuzzy1.googlecode.com?lang=go?path=alpha#%3Apkg%3A"},
{path: "github.com/google/kythe/foo", ticket: "kythe://github.com/google/kythe?lang=go?path=foo#%3Apkg%3A"},
{path: "bitbucket.org/zut/alors/non", ticket: "kythe://bitbucket.org/zut/alors?lang=go?path=non#%3Apkg%3A"},
{path: "launchpad.net/~frood/blee/blor", ticket: "kythe://launchpad.net/~frood/blee?lang=go?path=blor#%3Apkg%3A"},
{path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"},
}
for _, test := range tests {
pkg := &build.Package{
ImportPath: test.path,
Goroot: test.isRoot,
}
got := ForPackage("", pkg)
gotTicket := kytheuri.ToString(got)
if gotTicket != test.ticket {
t.Errorf(`ForPackage("", [%s]): got %q, want %q`, test.path, gotTicket, test.ticket)
}
}
}
func collectNodes(nodeEntries <-chan *spb.Entry) <-chan *srvpb.Node {
nodes := make(chan *srvpb.Node)
go func() {
var (
node *srvpb.Node
vname *spb.VName
)
for e := range nodeEntries {
if node != nil && !compare.VNamesEqual(e.Source, vname) {
nodes <- node
node = nil
vname = nil
}
if node == nil {
vname = e.Source
ticket := kytheuri.ToString(vname)
node = &srvpb.Node{Ticket: ticket}
}
node.Fact = append(node.Fact, &srvpb.Node_Fact{
Name: e.FactName,
Value: e.FactValue,
})
}
if node != nil {
nodes <- node
}
close(nodes)
}()
return nodes
}
// Sources constructs a new Source for every contiguous set of entries sharing
// the same Source, calling f for each.
func Sources(rd stream.EntryReader, f func(*ipb.Source) error) error {
var source *spb.VName
var src *ipb.Source
if err := rd(func(entry *spb.Entry) error {
if src != nil && !compare.VNamesEqual(source, entry.Source) {
if err := f(src); err != nil {
return err
}
src = nil
}
if src == nil {
source = entry.Source
src = &ipb.Source{
Ticket: kytheuri.ToString(entry.Source),
Facts: make(map[string][]byte),
EdgeGroups: make(map[string]*ipb.Source_EdgeGroup),
}
}
AppendEntry(src, entry)
return nil
}); err != nil {
return err
}
if src != nil {
return f(src)
}
return nil
}
// AppendEntry adds the given Entry to the Source's facts or edges. It is
// assumed that src.Ticket == kytheuri.ToString(e.Source).
func AppendEntry(src *ipb.Source, e *spb.Entry) {
if graphstore.IsEdge(e) {
kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
group, ok := src.EdgeGroups[kind]
if !ok {
group = &ipb.Source_EdgeGroup{}
src.EdgeGroups[kind] = group
}
ticket := kytheuri.ToString(e.Target)
ord := int32(ordinal)
for _, edge := range group.Edges {
if edge.Ticket == ticket && edge.Ordinal == ord {
// Don't add duplicate edge
return
}
}
group.Edges = append(group.Edges, &ipb.Source_Edge{
Ticket: ticket,
Ordinal: ord,
})
} else {
src.Facts[e.FactName] = e.FactValue
}
}
func (n *node) EdgeSet() *xpb.EdgeSet {
var groups []*xpb.EdgeSet_Group
for kind, targets := range n.Edges {
var tickets []string
for _, target := range targets {
tickets = append(tickets, kytheuri.ToString(target))
}
groups = append(groups, &xpb.EdgeSet_Group{
Kind: kind,
TargetTicket: tickets,
})
}
return &xpb.EdgeSet{
SourceTicket: kytheuri.ToString(n.Source),
Group: groups,
}
}
func writeEdges(ctx context.Context, t table.Proto, edges <-chan *spb.Entry, maxEdgePageSize int) error {
defer drainEntries(edges) // ensure channel is drained on errors
temp, err := tempTable("edge.groups")
if err != nil {
return fmt.Errorf("failed to create temporary table: %v", err)
}
edgeGroups := &table.KVProto{temp}
defer func() {
if err := edgeGroups.Close(ctx); err != nil {
log.Println("Error closing edge groups table: %v", err)
}
}()
log.Println("Writing temporary edges table")
var (
src *spb.VName
kind string
targets stringset.Set
)
for e := range edges {
if src != nil && (!compare.VNamesEqual(e.Source, src) || kind != e.EdgeKind) {
if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
return err
}
src = nil
}
if src == nil {
src = e.Source
kind = e.EdgeKind
targets = stringset.New()
}
targets.Add(kytheuri.ToString(e.Target))
}
if src != nil {
if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
return err
}
}
return writeEdgePages(ctx, t, edgeGroups, maxEdgePageSize)
}
func nodesToEdgeSets(nodes []*node) map[string]*gpb.EdgeSet {
sets := make(map[string]*gpb.EdgeSet)
for _, n := range nodes {
set := n.EdgeSet()
if len(set.Groups) > 0 {
sets[kytheuri.ToString(n.Source)] = set
}
}
return sets
}
func (n *node) Info() *xpb.NodeInfo {
info := &xpb.NodeInfo{
Ticket: kytheuri.ToString(n.Source),
}
for name, val := range n.Facts {
info.Fact = append(info.Fact, &xpb.Fact{
Name: name,
Value: []byte(val),
})
}
return info
}
func node(v *spb.VName, facts ...string) *srvpb.Node {
if len(facts)%2 != 0 {
panic("odd number of facts")
}
n := &srvpb.Node{Ticket: kytheuri.ToString(v)}
for i := 0; i < len(facts); i += 2 {
n.Fact = append(n.Fact, &srvpb.Node_Fact{
Name: facts[i],
Value: []byte(facts[i+1]),
})
}
return n
}
func TestDecorations(t *testing.T) {
xs := newService(t, testEntries)
reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
Location: &xpb.Location{
Ticket: kytheuri.ToString(testFileVName),
},
SourceText: true,
References: true,
})
if err != nil {
t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err)
}
if string(reply.SourceText) != testFileContent {
t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent)
}
if reply.Encoding != testFileEncoding {
t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding)
}
expectedRefs := []*xpb.DecorationsReply_Reference{
{
SourceTicket: kytheuri.ToString(testAnchorVName),
TargetTicket: kytheuri.ToString(testAnchorTargetVName),
Kind: schema.RefEdge,
},
}
if !reflect.DeepEqual(sortRefs(reply.Reference), sortRefs(expectedRefs)) {
t.Errorf("Got %v; Expected references %v", reply.Reference, expectedRefs)
}
refNodes := testNodes[4:6]
expectedNodes := nodesToInfos(refNodes)
if !reflect.DeepEqual(sortInfos(reply.Node), expectedNodes) {
t.Errorf("Got %v; Expected nodes %v", reply.Node, expectedNodes)
}
}
func (n *node) EdgeSet() *gpb.EdgeSet {
groups := make(map[string]*gpb.EdgeSet_Group)
for kind, targets := range n.Edges {
var edges []*gpb.EdgeSet_Group_Edge
for ordinal, target := range targets {
edges = append(edges, &gpb.EdgeSet_Group_Edge{
TargetTicket: kytheuri.ToString(target),
Ordinal: int32(ordinal),
})
}
groups[kind] = &gpb.EdgeSet_Group{
Edge: edges,
}
}
return &gpb.EdgeSet{
Groups: groups,
}
}
// SourceFromEntries returns a new Source from the given a set of entries with
// the same source VName.
func SourceFromEntries(entries []*spb.Entry) *ipb.Source {
if len(entries) == 0 {
return nil
}
src := &ipb.Source{
Ticket: kytheuri.ToString(entries[0].Source),
Facts: make(map[string][]byte),
EdgeGroups: make(map[string]*ipb.Source_EdgeGroup),
}
for _, e := range entries {
AppendEntry(src, e)
}
for _, group := range src.EdgeGroups {
sort.Sort(byOrdinal(group.Edges))
}
return src
}
// AddFile adds the given file VName to m.
func (m *Map) AddFile(file *spb.VName) {
ticket := kytheuri.ToString(file)
dirPath := CleanDirPath(path.Dir(file.Path))
dir := m.ensureDir(file.Corpus, file.Root, dirPath)
dir.File = addToSet(dir.File, ticket)
}
// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB) error {
log.Println("Starting serving pipeline")
tbl := &table.KVProto{db}
// TODO(schroederc): for large corpora, this won't fit in memory
var files []string
entries := make(chan *spb.Entry)
ftIn, nIn, eIn := make(chan *spb.VName), make(chan *spb.Entry), make(chan *spb.Entry)
go func() {
for entry := range entries {
if entry.EdgeKind == "" {
nIn <- entry
if entry.FactName == schema.NodeKindFact && string(entry.FactValue) == "file" {
ftIn <- entry.Source
files = append(files, kytheuri.ToString(entry.Source))
}
} else {
eIn <- entry
}
}
close(ftIn)
close(nIn)
close(eIn)
}()
log.Println("Scanning GraphStore")
var sErr error
go func() {
sErr = gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
entries <- e
return nil
})
close(entries)
}()
var (
ftErr, nErr, eErr error
ftWG, edgeNodeWG sync.WaitGroup
)
ftWG.Add(1)
go func() {
defer ftWG.Done()
ftErr = writeFileTree(ctx, tbl, ftIn)
log.Println("Wrote FileTree")
}()
edgeNodeWG.Add(2)
nodes := make(chan *srvpb.Node)
go func() {
defer edgeNodeWG.Done()
nErr = writeNodes(tbl, nIn, nodes)
log.Println("Wrote Nodes")
}()
go func() {
defer edgeNodeWG.Done()
eErr = writeEdges(ctx, tbl, eIn)
log.Println("Wrote Edges")
}()
var (
idxWG sync.WaitGroup
idxErr error
)
idxWG.Add(1)
go func() {
defer idxWG.Done()
idxErr = writeIndex(&table.KVInverted{db}, nodes)
log.Println("Wrote Search Index")
}()
edgeNodeWG.Wait()
if eErr != nil {
return eErr
} else if nErr != nil {
return nErr
}
es := xrefs.NodesEdgesService(&xsrv.Table{tbl})
if err := writeDecorations(ctx, tbl, es, files); err != nil {
return err
}
ftWG.Wait()
if ftErr != nil {
return ftErr
}
idxWG.Wait()
if idxErr != nil {
return idxErr
}
return sErr
}
func writeEdgePages(ctx context.Context, t table.Proto, gs graphstore.Service) error {
// TODO(schroederc): spill large PagedEdgeSets into EdgePages
log.Println("Writing EdgeSets")
var (
lastSrc *spb.VName
pes *srvpb.PagedEdgeSet
grp *srvpb.EdgeSet_Group
pesTotal int
)
if err := gs.Scan(ctx, new(spb.ScanRequest), func(e *spb.Entry) error {
if e.EdgeKind == "" {
panic("non-edge entry")
}
if pes != nil && !compare.VNamesEqual(lastSrc, e.Source) {
if grp != nil {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
}
pes.TotalEdges = int32(pesTotal)
if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
return err
}
pes = nil
grp = nil
pesTotal = 0
}
if pes == nil {
pes = &srvpb.PagedEdgeSet{
EdgeSet: &srvpb.EdgeSet{
SourceTicket: kytheuri.ToString(e.Source),
},
}
}
if grp != nil && grp.Kind != e.EdgeKind {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
grp = nil
}
if grp == nil {
grp = &srvpb.EdgeSet_Group{
Kind: e.EdgeKind,
}
}
grp.TargetTicket = append(grp.TargetTicket, kytheuri.ToString(e.Target))
lastSrc = e.Source
return nil
}); err != nil {
return err
}
if pes != nil {
if grp != nil {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
}
pes.TotalEdges = int32(pesTotal)
if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
return err
}
}
return nil
}
// Edges implements part of the Service interface.
func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) {
if len(req.Ticket) == 0 {
return nil, errors.New("no tickets specified")
} else if req.PageToken != "" {
return nil, errors.New("UNIMPLEMENTED: page_token")
}
patterns := xrefs.ConvertFilters(req.Filter)
allowedKinds := stringset.New(req.Kind...)
targetSet := stringset.New()
reply := new(xpb.EdgesReply)
for _, ticket := range req.Ticket {
vname, err := kytheuri.ToVName(ticket)
if err != nil {
return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err)
}
var (
// EdgeKind -> StringSet<TargetTicket>
filteredEdges = make(map[string]stringset.Set)
filteredFacts []*xpb.Fact
)
if err := g.gs.Read(ctx, &spb.ReadRequest{
Source: vname,
EdgeKind: "*",
}, func(entry *spb.Entry) error {
edgeKind := entry.EdgeKind
if edgeKind == "" {
// node fact
if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) {
filteredFacts = append(filteredFacts, entryToFact(entry))
}
} else {
// edge
if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) {
targets := filteredEdges[edgeKind]
if targets == nil {
targets = stringset.New()
filteredEdges[edgeKind] = targets
}
targets.Add(kytheuri.ToString(entry.Target))
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket)
}
// Only add a EdgeSet if there are targets for the requested edge kinds.
if len(filteredEdges) > 0 {
var groups []*xpb.EdgeSet_Group
for edgeKind, targets := range filteredEdges {
g := &xpb.EdgeSet_Group{Kind: edgeKind}
for target := range targets {
g.TargetTicket = append(g.TargetTicket, target)
targetSet.Add(target)
}
groups = append(groups, g)
}
reply.EdgeSet = append(reply.EdgeSet, &xpb.EdgeSet{
SourceTicket: ticket,
Group: groups,
})
// In addition, only add a NodeInfo if the filters have resulting facts.
if len(filteredFacts) > 0 {
reply.Node = append(reply.Node, &xpb.NodeInfo{
Ticket: ticket,
Fact: filteredFacts,
})
}
}
}
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Ensure reply.Node is a unique set by removing already requested nodes from targetSet
for _, n := range reply.Node {
targetSet.Remove(n.Ticket)
}
// Batch request all leftover target nodes
nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: targetSet.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting target nodes: %v", err)
}
reply.Node = append(reply.Node, nodesReply.Node...)
}
return reply, nil
}
func (d *DB) copyEntries(entries <-chan *spb.Entry) error {
// Start a transaction for a COPY statement per table
nodesTx, err := d.Begin()
if err != nil {
return err
}
edgesTx, err := d.Begin()
if err != nil {
return err
}
// Create each table in their corresponding transactions to speed up COPY
if _, err := nodesTx.Exec(createNodesTable); err != nil {
return fmt.Errorf("error truncating Nodes table: %v", err)
} else if _, err := edgesTx.Exec(createEdgeTable); err != nil {
return fmt.Errorf("error truncating Edges table: %v", err)
}
copyNode, err := nodesTx.Prepare(pq.CopyIn(
"nodes",
"ticket",
"node_kind",
"subkind",
"text",
"text_encoding",
"start_offset",
"end_offset",
"snippet_start",
"snippet_end",
"other_facts_num",
"other_facts",
))
if err != nil {
return fmt.Errorf("error preparing Nodes copy: %v", err)
}
copyEdge, err := edgesTx.Prepare(pq.CopyIn(
"edges",
"source",
"kind",
"target",
"ordinal",
))
if err != nil {
return fmt.Errorf("error preparing Edges copy: %v", err)
}
var node srvpb.Node
var nodeKind string
var subkind, textEncoding *string
var text *[]byte
var startOffset, endOffset, snippetStart, snippetEnd *int64
for e := range entries {
if graphstore.IsNodeFact(e) {
ticket := kytheuri.ToString(e.Source)
if node.Ticket != "" && node.Ticket != ticket {
nodeTicket := node.Ticket
node.Ticket = ""
var rec []byte
if len(node.Fact) > 0 {
rec, err = proto.Marshal(&node)
if err != nil {
return fmt.Errorf("error marshaling facts: %v", err)
}
}
if text != nil && textEncoding == nil {
textEncoding = proto.String(facts.DefaultTextEncoding)
}
if _, err := copyNode.Exec(
nodeTicket,
nodeKind,
subkind,
text,
textEncoding,
startOffset,
endOffset,
snippetStart,
snippetEnd,
len(node.Fact),
rec,
); err != nil {
return fmt.Errorf("error copying node: %v", err)
}
node.Fact, text = node.Fact[0:0], nil
nodeKind = ""
subkind, textEncoding = nil, nil
startOffset, endOffset, snippetStart, snippetEnd = nil, nil, nil, nil
}
if node.Ticket == "" {
node.Ticket = ticket
}
switch e.FactName {
case facts.NodeKind:
nodeKind = string(e.FactValue)
case facts.Subkind:
subkind = proto.String(string(e.FactValue))
case facts.Text:
text = &e.FactValue
case facts.TextEncoding:
textEncoding = proto.String(string(e.FactValue))
case facts.AnchorStart:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
startOffset = proto.Int64(n)
}
case facts.AnchorEnd:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
endOffset = proto.Int64(n)
}
case facts.SnippetStart:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
snippetStart = proto.Int64(n)
}
case facts.SnippetEnd:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
snippetEnd = proto.Int64(n)
}
default:
node.Fact = append(node.Fact, &cpb.Fact{
Name: e.FactName,
Value: e.FactValue,
})
}
} else if edges.IsForward(e.EdgeKind) {
kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
ticket := kytheuri.ToString(e.Source)
if _, err := copyEdge.Exec(ticket, kind, kytheuri.ToString(e.Target), ordinal); err != nil {
return fmt.Errorf("error copying edge: %v", err)
}
}
}
if _, err := copyNode.Exec(); err != nil {
return fmt.Errorf("error flushing nodes: %v", err)
} else if _, err := copyEdge.Exec(); err != nil {
return fmt.Errorf("error flushing edges: %v", err)
}
if err := nodesTx.Commit(); err != nil {
return fmt.Errorf("error committing Nodes transaction: %v", err)
} else if err := edgesTx.Commit(); err != nil {
return fmt.Errorf("error committing Edges transaction: %v", err)
}
return nil
}
// Edges implements part of the Service interface.
func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) {
if len(req.Ticket) == 0 {
return nil, errors.New("no tickets specified")
} else if req.PageToken != "" {
return nil, errors.New("UNIMPLEMENTED: page_token")
}
patterns := xrefs.ConvertFilters(req.Filter)
allowedKinds := stringset.New(req.Kind...)
targetSet := stringset.New()
reply := &xpb.EdgesReply{
EdgeSets: make(map[string]*xpb.EdgeSet),
Nodes: make(map[string]*xpb.NodeInfo),
}
for _, ticket := range req.Ticket {
vname, err := kytheuri.ToVName(ticket)
if err != nil {
return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err)
}
var (
// EdgeKind -> TargetTicket -> OrdinalSet
filteredEdges = make(map[string]map[string]map[int32]struct{})
filteredFacts = make(map[string][]byte)
)
if err := g.gs.Read(ctx, &spb.ReadRequest{
Source: vname,
EdgeKind: "*",
}, func(entry *spb.Entry) error {
edgeKind := entry.EdgeKind
if edgeKind == "" {
// node fact
if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) {
filteredFacts[entry.FactName] = entry.FactValue
}
} else {
// edge
edgeKind, ordinal, _ := schema.ParseOrdinal(edgeKind)
if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) {
targets, ok := filteredEdges[edgeKind]
if !ok {
targets = make(map[string]map[int32]struct{})
filteredEdges[edgeKind] = targets
}
ticket := kytheuri.ToString(entry.Target)
ordSet, ok := targets[ticket]
if !ok {
ordSet = make(map[int32]struct{})
targets[ticket] = ordSet
}
ordSet[int32(ordinal)] = struct{}{}
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket)
}
// Only add a EdgeSet if there are targets for the requested edge kinds.
if len(filteredEdges) > 0 {
groups := make(map[string]*xpb.EdgeSet_Group)
for edgeKind, targets := range filteredEdges {
g := &xpb.EdgeSet_Group{}
for target, ordinals := range targets {
for ordinal := range ordinals {
g.Edge = append(g.Edge, &xpb.EdgeSet_Group_Edge{
TargetTicket: target,
Ordinal: ordinal,
})
}
targetSet.Add(target)
}
groups[edgeKind] = g
}
reply.EdgeSets[ticket] = &xpb.EdgeSet{
Groups: groups,
}
// In addition, only add a NodeInfo if the filters have resulting facts.
if len(filteredFacts) > 0 {
reply.Nodes[ticket] = &xpb.NodeInfo{
Facts: filteredFacts,
}
}
}
}
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Eliminate redundant work by removing already requested nodes from targetSet
for ticket := range reply.Nodes {
targetSet.Remove(ticket)
}
// Batch request all leftover target nodes
nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: targetSet.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting target nodes: %v", err)
}
for ticket, node := range nodesReply.Nodes {
reply.Nodes[ticket] = node
}
}
return reply, nil
}
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if len(req.DirtyBuffer) > 0 {
return nil, errors.New("UNIMPLEMENTED: dirty buffers")
} else if req.GetLocation() == nil {
// TODO(schroederc): allow empty location when given dirty buffer
return nil, errors.New("missing location")
}
fileVName, err := kytheuri.ToVName(req.Location.Ticket)
if err != nil {
return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
}
text, encoding, err := getSourceText(ctx, g.gs, fileVName)
if err != nil {
return nil, fmt.Errorf("failed to retrieve file text: %v", err)
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{
Location: loc,
Nodes: make(map[string]*xpb.NodeInfo),
}
// Handle DecorationsRequest.SourceText switch
if req.SourceText {
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
reply.Encoding = encoding
}
// Handle DecorationsRequest.References switch
if req.References {
// Traverse the following chain of edges:
// file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
//
// Add []anchor and []target nodes to reply.Nodes
// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference
patterns := xrefs.ConvertFilters(req.Filter)
children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
return e.EdgeKind == revChildOfEdgeKind
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve file children: %v", err)
}
targetSet := stringset.New()
for _, edge := range children {
anchor := edge.Target
ticket := kytheuri.ToString(anchor)
anchorNodeReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: []string{ticket},
})
if err != nil {
return nil, fmt.Errorf("failure getting reference source node: %v", err)
} else if len(anchorNodeReply.Nodes) != 1 {
return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
}
node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
if !ok {
return nil, fmt.Errorf("failed to find info for node %q", ticket)
} else if string(node[schema.NodeKindFact]) != schema.AnchorKind {
// Skip child if it isn't an anchor node
continue
}
anchorStart, err := strconv.Atoi(string(node[schema.AnchorStartFact]))
if err != nil {
log.Printf("Invalid anchor start offset %q for node %q: %v", node[schema.AnchorStartFact], ticket, err)
continue
}
anchorEnd, err := strconv.Atoi(string(node[schema.AnchorEndFact]))
if err != nil {
log.Printf("Invalid anchor end offset %q for node %q: %v", node[schema.AnchorEndFact], ticket, err)
continue
}
if loc.Kind == xpb.Location_SPAN {
// Check if anchor fits within/around requested source text window
if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
continue
} else if anchorStart > anchorEnd {
log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
continue
}
}
targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
return schema.EdgeDirection(e.EdgeKind) == schema.Forward && e.EdgeKind != schema.ChildOfEdge
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
}
if len(targets) == 0 {
log.Printf("Anchor missing forward edges: {%+v}", anchor)
continue
}
if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
reply.Nodes[ticket] = node
}
for _, edge := range targets {
targetTicket := kytheuri.ToString(edge.Target)
targetSet.Add(targetTicket)
reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
SourceTicket: ticket,
Kind: edge.Kind,
TargetTicket: targetTicket,
AnchorStart: norm.ByteOffset(int32(anchorStart)),
AnchorEnd: norm.ByteOffset(int32(anchorEnd)),
})
}
}
sort.Sort(bySpan(reply.Reference))
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Ensure returned nodes are not duplicated.
for ticket := range reply.Nodes {
targetSet.Remove(ticket)
}
// Batch request all Reference target nodes
nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: targetSet.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
}
for ticket, node := range nodesReply.Nodes {
reply.Nodes[ticket] = node
}
}
}
return reply, nil
}
// AddFile adds the given file VName to m.
func (m *Map) AddFile(file *spb.VName) {
ticket := kytheuri.ToString(file)
path := filepath.Join("/", file.Path)
dir := m.ensureDir(file.Corpus, file.Root, filepath.Dir(path))
dir.File = addToSet(dir.File, ticket)
}