func TestDecorationsRefs(t *testing.T) {
d := tbl.Decorations[1]
st := tbl.Construct(t)
reply, err := st.Decorations(ctx, &xpb.DecorationsRequest{
Location: &xpb.Location{Ticket: d.FileTicket},
References: true,
Filter: []string{"**"},
})
testutil.FatalOnErrT(t, "DecorationsRequest error: %v", err)
if len(reply.SourceText) != 0 {
t.Errorf("Unexpected source text: %q", string(d.SourceText))
}
if reply.Encoding != "" {
t.Errorf("Unexpected encoding: %q", d.Encoding)
}
expected := refs(xrefs.NewNormalizer(d.SourceText), d.Decoration)
if !reflect.DeepEqual(expected, reply.Reference) {
t.Fatalf("Expected references %v; found %v", expected, reply.Reference)
}
expectedNodes := nodeInfos(tbl.Nodes[7:13])
sort.Sort(byNodeTicket(expectedNodes))
sort.Sort(byNodeTicket(reply.Node))
if err := testutil.DeepEqual(expectedNodes, reply.Node); err != nil {
t.Fatal(err)
}
}
// Decorations implements part of the xrefs.Interface.
func (d *DB) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if d.selectText == nil {
var err error
d.selectText, err = d.Prepare("SELECT text, text_encoding FROM Nodes WHERE ticket = $1;")
if err != nil {
return nil, fmt.Errorf("error preparing selectText statement: %v", err)
}
}
// TODO(schroederc): dirty buffers
// TODO(schroederc): span locations
fileTicket, err := kytheuri.Fix(req.Location.Ticket)
if err != nil {
return nil, fmt.Errorf("invalid location ticket: %v", err)
}
req.Location.Ticket = fileTicket
decor := &xpb.DecorationsReply{Location: req.Location}
r := d.selectText.QueryRow(fileTicket)
var text []byte
var textEncoding sql.NullString
if err := r.Scan(&text, &textEncoding); err != nil {
return nil, err
}
norm := xrefs.NewNormalizer(text)
if req.SourceText {
decor.SourceText = text
decor.Encoding = textEncoding.String
}
if req.References {
var err error
decor.Reference, err = d.scanReferences(fileTicket, norm)
if err != nil {
return nil, err
}
if len(req.Filter) > 0 && len(decor.Reference) > 0 {
nodeTickets := stringset.New()
for _, r := range decor.Reference {
nodeTickets.Add(r.TargetTicket)
}
nodes, err := d.Nodes(ctx, &xpb.NodesRequest{
Ticket: nodeTickets.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("error filtering nodes:%v", err)
}
decor.Nodes = nodes.Nodes
}
}
return decor, nil
}
func normalizedPoint(text []byte) int {
p := xrefs.NewNormalizer(text).Point(&xpb.Location_Point{
ByteOffset: max(*offset, 0),
LineNumber: max(*lineNumber, 0),
ColumnOffset: max(*columnOffset, 0),
})
return int(p.ByteOffset)
}
func TestDecorationsDirtyBuffer(t *testing.T) {
d := tbl.Decorations[1]
st := tbl.Construct(t)
dirty := []byte(`(defn map [f coll]
(if (seq coll)
[]
(cons (f (first coll)) (map f (rest coll)))))
`)
reply, err := st.Decorations(ctx, &xpb.DecorationsRequest{
Location: &xpb.Location{Ticket: d.FileTicket},
DirtyBuffer: dirty,
References: true,
Filter: []string{"**"},
})
testutil.FatalOnErrT(t, "DecorationsRequest error: %v", err)
if len(reply.SourceText) != 0 {
t.Errorf("Unexpected source text: %q", string(d.SourceText))
}
if reply.Encoding != "" {
t.Errorf("Unexpected encoding: %q", d.Encoding)
}
p := xrefs.NewPatcher(d.SourceText, dirty)
norm := xrefs.NewNormalizer(dirty)
var expected []*xpb.DecorationsReply_Reference
for _, d := range d.Decoration {
if _, _, exists := p.Patch(d.Anchor.StartOffset, d.Anchor.EndOffset); exists {
expected = append(expected, decorationToReference(norm, d))
}
}
if !reflect.DeepEqual(expected, reply.Reference) {
t.Fatalf("Expected references %v; found %v", expected, reply.Reference)
}
// These are a subset of the anchor nodes in tbl.Decorations[1]. tbl.Nodes[8]
// and tbl.Nodes[10] are missing because [8] was an anchor in the edited
// region and [10] was its target.
expectedNodes := nodeInfos([]*srvpb.Node{
tbl.Nodes[7], tbl.Nodes[9], tbl.Nodes[11], tbl.Nodes[12],
})
// Ensure patching affects the anchor node facts
mapFacts(expectedNodes[2], map[string]string{
"/kythe/loc/start": "48",
"/kythe/loc/end": "52",
})
sort.Sort(byNodeTicket(expectedNodes))
sort.Sort(byNodeTicket(reply.Node))
if !reflect.DeepEqual(expectedNodes, reply.Node) {
t.Fatalf("Expected nodes %v; found %v", expected, reply.Node)
}
}
func displayReferences(decor *xpb.DecorationsReply) error {
if *displayJSON {
return json.NewEncoder(out).Encode(decor)
}
nodes := xrefs.NodesMap(decor.Node)
// TODO(schroederc): return anchor locations from server
norm := xrefs.NewNormalizer(decor.SourceText)
for _, ref := range decor.Reference {
nodeKind := factValue(nodes, ref.TargetTicket, schema.NodeKindFact, "UNKNOWN")
subkind := factValue(nodes, ref.TargetTicket, schema.SubkindFact, "")
startOffset := factValue(nodes, ref.SourceTicket, schema.AnchorStartFact, "_")
endOffset := factValue(nodes, ref.SourceTicket, schema.AnchorEndFact, "_")
// ignore errors (locations will be 0)
s, _ := strconv.Atoi(startOffset)
e, _ := strconv.Atoi(endOffset)
loc, err := norm.Location(&xpb.Location{
Kind: xpb.Location_SPAN,
Start: &xpb.Location_Point{ByteOffset: int32(s)},
End: &xpb.Location_Point{ByteOffset: int32(e)},
})
if err != nil {
return fmt.Errorf("error normalizing reference location for anchor %q: %v", ref.SourceTicket, err)
}
r := strings.NewReplacer(
"@source@", ref.SourceTicket,
"@target@", ref.TargetTicket,
"@edgeKind@", ref.Kind,
"@nodeKind@", nodeKind,
"@subkind@", subkind,
"@^offset@", startOffset,
"@^line@", strconv.Itoa(int(loc.Start.LineNumber)),
"@^col@", strconv.Itoa(int(loc.Start.ColumnOffset)),
"@$offset@", endOffset,
"@$line@", strconv.Itoa(int(loc.End.LineNumber)),
"@$col@", strconv.Itoa(int(loc.End.ColumnOffset)),
)
if _, err := r.WriteString(out, refFormat+"\n"); err != nil {
return err
}
}
return nil
}
// Decorations implements part of the xrefs Service interface.
func (t *tableImpl) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if req.GetLocation() == nil || req.GetLocation().Ticket == "" {
return nil, errors.New("missing location")
}
ticket, err := kytheuri.Fix(req.GetLocation().Ticket)
if err != nil {
return nil, fmt.Errorf("invalid ticket %q: %v", req.GetLocation().Ticket, err)
}
decor, err := t.fileDecorations(ctx, ticket)
if err == table.ErrNoSuchKey {
return nil, xrefs.ErrDecorationsNotFound
} else if err != nil {
return nil, fmt.Errorf("lookup error for file decorations %q: %v", ticket, err)
}
text := decor.File.Text
if len(req.DirtyBuffer) > 0 {
text = req.DirtyBuffer
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{Location: loc}
if req.SourceText {
reply.Encoding = decor.File.Encoding
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
}
if req.References {
patterns := xrefs.ConvertFilters(req.Filter)
var patcher *xrefs.Patcher
if len(req.DirtyBuffer) > 0 {
patcher = xrefs.NewPatcher(decor.File.Text, req.DirtyBuffer)
}
// The span with which to constrain the set of returned anchor references.
var startBoundary, endBoundary int32
spanKind := req.SpanKind
if loc.Kind == xpb.Location_FILE {
startBoundary = 0
endBoundary = int32(len(text))
spanKind = xpb.DecorationsRequest_WITHIN_SPAN
} else {
startBoundary = loc.Start.ByteOffset
endBoundary = loc.End.ByteOffset
}
reply.Reference = make([]*xpb.DecorationsReply_Reference, 0, len(decor.Decoration))
reply.Nodes = make(map[string]*xpb.NodeInfo)
seenTarget := stringset.New()
// Reference.TargetTicket -> NodeInfo (superset of reply.Nodes)
var nodes map[string]*xpb.NodeInfo
if len(patterns) > 0 {
nodes = make(map[string]*xpb.NodeInfo)
for _, n := range decor.Target {
nodes[n.Ticket] = nodeToInfo(patterns, n)
}
}
// Reference.TargetTicket -> []Reference set
var refs map[string][]*xpb.DecorationsReply_Reference
// ExpandedAnchor.Ticket -> ExpandedAnchor
var defs map[string]*srvpb.ExpandedAnchor
if req.TargetDefinitions {
refs = make(map[string][]*xpb.DecorationsReply_Reference)
reply.DefinitionLocations = make(map[string]*xpb.Anchor)
defs = make(map[string]*srvpb.ExpandedAnchor)
for _, def := range decor.TargetDefinitions {
defs[def.Ticket] = def
}
}
for _, d := range decor.Decoration {
start, end, exists := patcher.Patch(d.Anchor.StartOffset, d.Anchor.EndOffset)
// Filter non-existent anchor. Anchors can no longer exist if we were
// given a dirty buffer and the anchor was inside a changed region.
if exists {
if xrefs.InSpanBounds(spanKind, start, end, startBoundary, endBoundary) {
d.Anchor.StartOffset = start
d.Anchor.EndOffset = end
r := decorationToReference(norm, d)
if req.TargetDefinitions {
if def, ok := defs[d.TargetDefinition]; ok {
reply.DefinitionLocations[d.TargetDefinition] = a2a(def, false).Anchor
} else {
refs[r.TargetTicket] = append(refs[r.TargetTicket], r)
}
} else {
r.TargetDefinition = ""
}
reply.Reference = append(reply.Reference, r)
if !seenTarget.Contains(r.TargetTicket) && nodes != nil {
reply.Nodes[r.TargetTicket] = nodes[r.TargetTicket]
seenTarget.Add(r.TargetTicket)
}
}
}
}
// Only compute target definitions if the serving data doesn't contain any
// TODO(schroederc): remove this once serving data is always populated
if req.TargetDefinitions && len(defs) == 0 {
targetTickets := make([]string, 0, len(refs))
for ticket := range refs {
targetTickets = append(targetTickets, ticket)
}
defs, err := xrefs.SlowDefinitions(t, ctx, targetTickets)
if err != nil {
return nil, fmt.Errorf("error retrieving target definitions: %v", err)
}
reply.DefinitionLocations = make(map[string]*xpb.Anchor, len(defs))
for tgt, def := range defs {
for _, ref := range refs[tgt] {
if def.Ticket != ref.SourceTicket {
ref.TargetDefinition = def.Ticket
if _, ok := reply.DefinitionLocations[def.Ticket]; !ok {
reply.DefinitionLocations[def.Ticket] = def
}
}
}
}
}
}
return reply, nil
}
func (d *DB) createCrossRefsTable() error {
if _, err := d.Exec(createCrossReferencesTable); err != nil {
return fmt.Errorf("error creating cross-references table: %v", err)
}
txn, err := d.Begin()
if err != nil {
return fmt.Errorf("error creating transaction: %v", err)
}
copyXRef, err := txn.Prepare(pq.CopyIn("crossreferences", "ticket", "kind", "file_ticket", "anchor_ticket", "proto"))
if err != nil {
return fmt.Errorf("error preparing CrossReferences copy statement: %v", err)
}
rs, err := d.Query(`SELECT decor.target_ticket, decor.kind, decor.file_ticket, decor.anchor_ticket, anchor.start_offset, anchor.end_offset, anchor.snippet_start, anchor.snippet_end
FROM Decorations decor
JOIN Nodes anchor ON anchor.ticket = decor.anchor_ticket
ORDER BY file_ticket;`)
if err != nil {
return fmt.Errorf("error creating decorations query: %v", err)
}
queryFile, err := d.Prepare("SELECT text, text_encoding FROM Nodes WHERE ticket = $1;")
if err != nil {
return fmt.Errorf("error preparing file query: %v", err)
}
var (
file srvpb.File
raw srvpb.RawAnchor
norm *xrefs.Normalizer
lastFile string
)
for rs.Next() {
var ticket, kind string
var snippetStart, snippetEnd sql.NullInt64
if err := rs.Scan(&ticket, &kind, &file.Ticket, &raw.Ticket, &raw.StartOffset, &raw.EndOffset, &snippetStart, &snippetEnd); err != nil {
return fmt.Errorf("Decorations scan error: %v", err)
}
if snippetStart.Valid {
raw.SnippetStart = int32(snippetStart.Int64)
} else {
raw.SnippetStart = 0
}
if snippetEnd.Valid {
raw.SnippetEnd = int32(snippetEnd.Int64)
} else {
raw.SnippetEnd = 0
}
if lastFile != file.Ticket {
var textEncoding sql.NullString
if err := queryFile.QueryRow(file.Ticket).Scan(&file.Text, &textEncoding); err != nil {
return fmt.Errorf("error looking up file: %v", err)
}
file.Encoding = textEncoding.String
norm = xrefs.NewNormalizer(file.Text)
lastFile = file.Ticket
}
a, err := assemble.ExpandAnchor(&raw, &file, norm, kind)
if err != nil {
return fmt.Errorf("error expanding anchor: %v", err)
}
rec, err := proto.Marshal(a2a(a, true))
if err != nil {
return fmt.Errorf("error marshaling anchor: %v", err)
}
if _, err := copyXRef.Exec(ticket, kind, file.Ticket, raw.Ticket, rec); err != nil {
return fmt.Errorf("copy error: %v", err)
}
}
if _, err := copyXRef.Exec(); err != nil {
return fmt.Errorf("error flushing CrossReferences: %v", err)
}
if err := txn.Commit(); err != nil {
return fmt.Errorf("transaction commit error: %v", err)
}
return nil
}
// Decorations implements part of the xrefs Service interface.
func (t *tableImpl) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if req.GetLocation() == nil || req.GetLocation().Ticket == "" {
return nil, errors.New("missing location")
}
ticket, err := kytheuri.Fix(req.GetLocation().Ticket)
if err != nil {
return nil, fmt.Errorf("invalid ticket %q: %v", req.GetLocation().Ticket, err)
}
decor, err := t.fileDecorations(ctx, ticket)
if err == table.ErrNoSuchKey {
return nil, fmt.Errorf("decorations not found for file %q", ticket)
} else if err != nil {
return nil, fmt.Errorf("lookup error for file decorations %q: %v", ticket, err)
}
text := decor.SourceText
if len(req.DirtyBuffer) > 0 {
text = req.DirtyBuffer
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{Location: loc}
if req.SourceText {
reply.Encoding = decor.Encoding
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
}
if req.References {
// Set of node tickets for which to retrieve facts. These are the nodes
// used in the returned references (both anchor sources and node targets).
nodeTickets := stringset.New()
var patcher *xrefs.Patcher
var offsetMapping map[string]span // Map from anchor ticket to patched span
if len(req.DirtyBuffer) > 0 {
patcher = xrefs.NewPatcher(decor.SourceText, req.DirtyBuffer)
offsetMapping = make(map[string]span)
}
// The span with which to constrain the set of returned anchor references.
var startBoundary, endBoundary int32
if loc.Kind == xpb.Location_FILE {
startBoundary = 0
endBoundary = int32(len(text))
} else {
startBoundary = loc.Start.ByteOffset
endBoundary = loc.End.ByteOffset
}
reply.Reference = make([]*xpb.DecorationsReply_Reference, 0, len(decor.Decoration))
for _, d := range decor.Decoration {
start, end, exists := patcher.Patch(d.Anchor.StartOffset, d.Anchor.EndOffset)
// Filter non-existent anchor. Anchors can no longer exist if we were
// given a dirty buffer and the anchor was inside a changed region.
if exists {
if start >= startBoundary && end <= endBoundary {
if offsetMapping != nil {
// Save the patched span to update the corresponding facts of the
// anchor node in reply.Node.
offsetMapping[d.Anchor.Ticket] = span{start, end}
}
reply.Reference = append(reply.Reference, decorationToReference(norm, d))
nodeTickets.Add(d.Anchor.Ticket)
nodeTickets.Add(d.TargetTicket)
}
}
}
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Retrieve facts for all nodes referenced in the file decorations.
nodesReply, err := t.Nodes(ctx, &xpb.NodesRequest{
Ticket: nodeTickets.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("error getting nodes: %v", err)
}
reply.Node = nodesReply.Node
}
// Patch anchor node facts in reply to match dirty buffer
if len(offsetMapping) > 0 {
for _, n := range reply.Node {
if span, ok := offsetMapping[n.Ticket]; ok {
for _, f := range n.Fact {
switch f.Name {
case schema.AnchorStartFact:
f.Value = []byte(strconv.Itoa(int(span.start)))
case schema.AnchorEndFact:
f.Value = []byte(strconv.Itoa(int(span.end)))
}
}
}
}
}
}
return reply, nil
}
func writeDecorAndRefs(ctx context.Context, opts *Options, edges <-chan *srvpb.Edge, out *servingOutput) error {
fragments, err := opts.diskSorter(fragmentLesser{}, fragmentMarshaler{})
if err != nil {
return err
}
log.Println("Writing decoration fragments")
if err := createDecorationFragments(ctx, edges, fragments); err != nil {
return err
}
log.Println("Writing completed FileDecorations")
// refSorter stores a *srvpb.CrossReference for each Decoration from fragments
refSorter, err := opts.diskSorter(refLesser{}, refMarshaler{})
if err != nil {
return fmt.Errorf("error creating sorter: %v", err)
}
buffer := out.xs.Buffered()
var (
curFile string
file *srvpb.File
norm *xrefs.Normalizer
decor *srvpb.FileDecorations
)
if err := fragments.Read(func(x interface{}) error {
df := x.(*decorationFragment)
fileTicket := df.fileTicket
fragment := df.decoration
if decor != nil && curFile != fileTicket {
if decor.File != nil {
if err := writeDecor(ctx, buffer, decor); err != nil {
return err
}
file = nil
}
decor = nil
}
curFile = fileTicket
if decor == nil {
decor = &srvpb.FileDecorations{}
}
if fragment.File == nil {
decor.Decoration = append(decor.Decoration, fragment.Decoration...)
if file == nil {
return errors.New("missing file for anchors")
}
// Reverse each fragment.Decoration to create a *srvpb.CrossReference
for _, d := range fragment.Decoration {
cr, err := assemble.CrossReference(file, norm, d)
if err != nil {
return fmt.Errorf("error assembling cross-reference: %v", err)
}
if err := refSorter.Add(cr); err != nil {
return fmt.Errorf("error adding CrossReference to sorter: %v", err)
}
}
} else {
decor.File = fragment.File
file = fragment.File
norm = xrefs.NewNormalizer(file.Text)
}
return nil
}); err != nil {
return fmt.Errorf("error reading decoration fragments: %v", err)
}
if decor != nil && decor.File != nil {
if err := writeDecor(ctx, buffer, decor); err != nil {
return err
}
}
log.Println("Writing CrossReferences")
xb := &assemble.CrossReferencesBuilder{
MaxPageSize: opts.MaxPageSize,
Output: func(ctx context.Context, s *srvpb.PagedCrossReferences) error {
return buffer.Put(ctx, xsrv.CrossReferencesKey(s.SourceTicket), s)
},
OutputPage: func(ctx context.Context, p *srvpb.PagedCrossReferences_Page) error {
return buffer.Put(ctx, xsrv.CrossReferencesPageKey(p.PageKey), p)
},
}
var curTicket string
if err := refSorter.Read(func(i interface{}) error {
cr := i.(*srvpb.CrossReference)
if curTicket != cr.Referent.Ticket {
curTicket = cr.Referent.Ticket
if err := xb.StartSet(ctx, curTicket); err != nil {
return fmt.Errorf("error starting cross-references set: %v", err)
}
}
g := &srvpb.PagedCrossReferences_Group{
Kind: cr.TargetAnchor.Kind,
Anchor: []*srvpb.ExpandedAnchor{cr.TargetAnchor},
}
if err := xb.AddGroup(ctx, g); err != nil {
return fmt.Errorf("error adding cross-reference: %v", err)
}
return nil
}); err != nil {
return fmt.Errorf("error reading xrefs: %v", err)
}
if err := xb.Flush(ctx); err != nil {
return fmt.Errorf("error flushing cross-references: %v", err)
}
return buffer.Flush(ctx)
}
func completeAnchors(ctx context.Context, xs xrefs.NodesEdgesService, retrieveText bool, files map[string]*fileNode, edgeKind string, anchors []string) ([]*xpb.CrossReferencesReply_RelatedAnchor, error) {
edgeKind = schema.Canonicalize(edgeKind)
// AllEdges is relatively safe because each anchor will have very few parents (almost always 1)
reply, err := xrefs.AllEdges(ctx, xs, &xpb.EdgesRequest{
Ticket: anchors,
Kind: []string{schema.ChildOfEdge},
Filter: []string{
schema.NodeKindFact,
schema.AnchorLocFilter,
schema.SnippetLocFilter,
},
})
if err != nil {
return nil, err
}
nodes := xrefs.NodesMap(reply.Nodes)
var result []*xpb.CrossReferencesReply_RelatedAnchor
for ticket, es := range reply.EdgeSets {
if nodeKind := string(nodes[ticket][schema.NodeKindFact]); nodeKind != schema.AnchorKind {
log.Printf("Found non-anchor target to %q edge: %q (kind: %q)", edgeKind, ticket, nodeKind)
continue
}
// Parse anchor location start/end facts
so, eo, err := getSpan(nodes[ticket], schema.AnchorStartFact, schema.AnchorEndFact)
if err != nil {
log.Printf("Invalid anchor span for %q: %v", ticket, err)
continue
}
// For each file parent to the anchor, add an Anchor to the result.
for kind, g := range es.Groups {
if kind != schema.ChildOfEdge {
continue
}
for _, edge := range g.Edge {
parent := edge.TargetTicket
if parentKind := string(nodes[parent][schema.NodeKindFact]); parentKind != schema.FileKind {
log.Printf("Found non-file parent to anchor: %q (kind: %q)", parent, parentKind)
continue
}
a := &xpb.Anchor{
Ticket: ticket,
Kind: edgeKind,
Parent: parent,
}
file, ok := files[a.Parent]
if !ok {
nReply, err := xs.Nodes(ctx, &xpb.NodesRequest{Ticket: []string{a.Parent}})
if err != nil {
return nil, fmt.Errorf("error getting file contents for %q: %v", a.Parent, err)
}
nMap := xrefs.NodesMap(nReply.Nodes)
text := nMap[a.Parent][schema.TextFact]
file = &fileNode{
text: text,
encoding: string(nMap[a.Parent][schema.TextEncodingFact]),
norm: xrefs.NewNormalizer(text),
}
files[a.Parent] = file
}
a.Start, a.End, err = normalizeSpan(file.norm, int32(so), int32(eo))
if err != nil {
log.Printf("Invalid anchor span %q in file %q: %v", ticket, a.Parent, err)
continue
}
if retrieveText && a.Start.ByteOffset < a.End.ByteOffset {
a.Text, err = text.ToUTF8(file.encoding, file.text[a.Start.ByteOffset:a.End.ByteOffset])
if err != nil {
log.Printf("Error decoding anchor text: %v", err)
}
}
if snippetStart, snippetEnd, err := getSpan(nodes[ticket], schema.SnippetStartFact, schema.SnippetEndFact); err == nil {
startPoint, endPoint, err := normalizeSpan(file.norm, int32(snippetStart), int32(snippetEnd))
if err != nil {
log.Printf("Invalid snippet span %q in file %q: %v", ticket, a.Parent, err)
} else {
a.Snippet, err = text.ToUTF8(file.encoding, file.text[startPoint.ByteOffset:endPoint.ByteOffset])
if err != nil {
log.Printf("Error decoding snippet text: %v", err)
}
a.SnippetStart = startPoint
a.SnippetEnd = endPoint
}
}
// fallback to a line-based snippet if the indexer did not provide its own snippet offsets
if a.Snippet == "" {
a.SnippetStart = &xpb.Location_Point{
ByteOffset: a.Start.ByteOffset - a.Start.ColumnOffset,
LineNumber: a.Start.LineNumber,
}
nextLine := file.norm.Point(&xpb.Location_Point{LineNumber: a.Start.LineNumber + 1})
a.SnippetEnd = &xpb.Location_Point{
ByteOffset: nextLine.ByteOffset - 1,
LineNumber: a.Start.LineNumber,
ColumnOffset: a.Start.ColumnOffset + (nextLine.ByteOffset - a.Start.ByteOffset - 1),
}
a.Snippet, err = text.ToUTF8(file.encoding, file.text[a.SnippetStart.ByteOffset:a.SnippetEnd.ByteOffset])
if err != nil {
log.Printf("Error decoding snippet text: %v", err)
}
}
result = append(result, &xpb.CrossReferencesReply_RelatedAnchor{Anchor: a})
}
break // we've handled the only /kythe/edge/childof group
}
}
return result, nil
}
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if len(req.DirtyBuffer) > 0 {
return nil, errors.New("UNIMPLEMENTED: dirty buffers")
} else if req.GetLocation() == nil {
// TODO(schroederc): allow empty location when given dirty buffer
return nil, errors.New("missing location")
}
fileVName, err := kytheuri.ToVName(req.Location.Ticket)
if err != nil {
return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
}
text, encoding, err := getSourceText(ctx, g.gs, fileVName)
if err != nil {
return nil, fmt.Errorf("failed to retrieve file text: %v", err)
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{
Location: loc,
Nodes: make(map[string]*xpb.NodeInfo),
}
// Handle DecorationsRequest.SourceText switch
if req.SourceText {
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
reply.Encoding = encoding
}
// Handle DecorationsRequest.References switch
if req.References {
// Traverse the following chain of edges:
// file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
//
// Add []anchor and []target nodes to reply.Nodes
// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference
patterns := xrefs.ConvertFilters(req.Filter)
children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
return e.EdgeKind == revChildOfEdgeKind
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve file children: %v", err)
}
targetSet := stringset.New()
for _, edge := range children {
anchor := edge.Target
ticket := kytheuri.ToString(anchor)
anchorNodeReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: []string{ticket},
})
if err != nil {
return nil, fmt.Errorf("failure getting reference source node: %v", err)
} else if len(anchorNodeReply.Nodes) != 1 {
return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
}
node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
if !ok {
return nil, fmt.Errorf("failed to find info for node %q", ticket)
} else if string(node[schema.NodeKindFact]) != schema.AnchorKind {
// Skip child if it isn't an anchor node
continue
}
anchorStart, err := strconv.Atoi(string(node[schema.AnchorStartFact]))
if err != nil {
log.Printf("Invalid anchor start offset %q for node %q: %v", node[schema.AnchorStartFact], ticket, err)
continue
}
anchorEnd, err := strconv.Atoi(string(node[schema.AnchorEndFact]))
if err != nil {
log.Printf("Invalid anchor end offset %q for node %q: %v", node[schema.AnchorEndFact], ticket, err)
continue
}
if loc.Kind == xpb.Location_SPAN {
// Check if anchor fits within/around requested source text window
if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
continue
} else if anchorStart > anchorEnd {
log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
continue
}
}
targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
return schema.EdgeDirection(e.EdgeKind) == schema.Forward && e.EdgeKind != schema.ChildOfEdge
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
}
if len(targets) == 0 {
log.Printf("Anchor missing forward edges: {%+v}", anchor)
continue
}
if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
reply.Nodes[ticket] = node
}
for _, edge := range targets {
targetTicket := kytheuri.ToString(edge.Target)
targetSet.Add(targetTicket)
reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
SourceTicket: ticket,
Kind: edge.Kind,
TargetTicket: targetTicket,
AnchorStart: norm.ByteOffset(int32(anchorStart)),
AnchorEnd: norm.ByteOffset(int32(anchorEnd)),
})
}
}
sort.Sort(bySpan(reply.Reference))
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Ensure returned nodes are not duplicated.
for ticket := range reply.Nodes {
targetSet.Remove(ticket)
}
// Batch request all Reference target nodes
nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
Ticket: targetSet.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
}
for ticket, node := range nodesReply.Nodes {
reply.Nodes[ticket] = node
}
}
}
return reply, nil
}
// Decorations implements part of the xrefs Service interface.
func (t *tableImpl) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if req.GetLocation() == nil || req.GetLocation().Ticket == "" {
return nil, errors.New("missing location")
}
ticket, err := kytheuri.Fix(req.GetLocation().Ticket)
if err != nil {
return nil, fmt.Errorf("invalid ticket %q: %v", req.GetLocation().Ticket, err)
}
decor, err := t.fileDecorations(ctx, ticket)
if err == table.ErrNoSuchKey {
return nil, fmt.Errorf("decorations not found for file %q", ticket)
} else if err != nil {
return nil, fmt.Errorf("lookup error for file decorations %q: %v", ticket, err)
}
text := decor.SourceText
if len(req.DirtyBuffer) > 0 {
text = req.DirtyBuffer
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{Location: loc}
if req.SourceText {
reply.Encoding = decor.Encoding
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
}
if req.References {
patterns := xrefs.ConvertFilters(req.Filter)
nodeTickets := stringset.New()
var patcher *xrefs.Patcher
var offsetMapping map[string]span // Map from anchor ticket to patched span
if len(req.DirtyBuffer) > 0 {
patcher = xrefs.NewPatcher(decor.SourceText, req.DirtyBuffer)
offsetMapping = make(map[string]span)
}
// The span with which to constrain the set of returned anchor references.
var startBoundary, endBoundary int32
if loc.Kind == xpb.Location_FILE {
startBoundary = 0
endBoundary = int32(len(text))
} else {
startBoundary = loc.Start.ByteOffset
endBoundary = loc.End.ByteOffset
}
reply.Reference = make([]*xpb.DecorationsReply_Reference, 0, len(decor.Decoration))
for _, d := range decor.Decoration {
start, end, exists := patcher.Patch(d.Anchor.StartOffset, d.Anchor.EndOffset)
// Filter non-existent anchor. Anchors can no longer exist if we were
// given a dirty buffer and the anchor was inside a changed region.
if exists {
if start >= startBoundary && end <= endBoundary {
if offsetMapping != nil {
// Save the patched span to update the corresponding facts of the
// anchor node in reply.Node.
offsetMapping[d.Anchor.Ticket] = span{start, end}
}
reply.Reference = append(reply.Reference, decorationToReference(norm, d))
if len(patterns) > 0 && !nodeTickets.Contains(d.Target.Ticket) {
nodeTickets.Add(d.Target.Ticket)
reply.Node = append(reply.Node, nodeToInfo(patterns, d.Target))
}
}
}
}
}
return reply, nil
}
// Decorations implements part of the xrefs Service interface.
func (t *Table) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if len(req.DirtyBuffer) > 0 {
log.Println("TODO: implement DecorationsRequest.DirtyBuffer")
return nil, errors.New("dirty buffers unimplemented")
} else if req.GetLocation() == nil {
// TODO(schroederc): allow empty location when given dirty buffer
return nil, errors.New("missing location")
}
var decor srvpb.FileDecorations
ticket := req.GetLocation().Ticket
if err := t.Lookup(DecorationsKey(ticket), &decor); err == table.ErrNoSuchKey {
return nil, fmt.Errorf("decorations not found for file %q", ticket)
} else if err != nil {
return nil, fmt.Errorf("lookup error for file decorations %q: %v", ticket, err)
}
loc, err := xrefs.NewNormalizer(decor.SourceText).Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{Location: loc}
if req.SourceText {
reply.Encoding = decor.Encoding
if loc.Kind == xpb.Location_FILE {
reply.SourceText = decor.SourceText
} else {
reply.SourceText = decor.SourceText[loc.Start.ByteOffset:loc.End.ByteOffset]
}
}
if req.References {
nodeTickets := stringset.New()
if loc.Kind == xpb.Location_FILE {
reply.Reference = make([]*xpb.DecorationsReply_Reference, len(decor.Decoration))
for i, d := range decor.Decoration {
reply.Reference[i] = decorationToReference(d)
nodeTickets.Add(d.Anchor.Ticket)
nodeTickets.Add(d.TargetTicket)
}
} else {
for _, d := range decor.Decoration {
if d.Anchor.StartOffset >= loc.Start.ByteOffset && d.Anchor.EndOffset <= loc.End.ByteOffset {
reply.Reference = append(reply.Reference, decorationToReference(d))
nodeTickets.Add(d.Anchor.Ticket)
nodeTickets.Add(d.TargetTicket)
}
}
}
nodesReply, err := t.Nodes(ctx, &xpb.NodesRequest{Ticket: nodeTickets.Slice()})
if err != nil {
return nil, fmt.Errorf("error getting nodes: %v", err)
}
reply.Node = nodesReply.Node
}
return reply, nil
}
func displayDecorations(text []byte, decor *xpb.DecorationsReply) error {
if *displayJSON {
return json.NewEncoder(out).Encode(decor)
}
nodes := xrefs.NodesMap(decor.Node)
if text == nil {
text = decor.SourceText
}
norm := xrefs.NewNormalizer(text)
for _, ref := range decor.Reference {
nodeKind := factValue(nodes, ref.TargetTicket, schema.NodeKindFact, "UNKNOWN")
subkind := factValue(nodes, ref.TargetTicket, schema.SubkindFact, "")
var loc *xpb.Location
if ref.AnchorStart != nil && ref.AnchorEnd != nil {
loc = &xpb.Location{
Kind: xpb.Location_SPAN,
Start: ref.AnchorStart,
End: ref.AnchorEnd,
}
} else {
// TODO(schroederc): remove this backwards-compatibility branch
startOffset := factValue(nodes, ref.SourceTicket, schema.AnchorStartFact, "_")
endOffset := factValue(nodes, ref.SourceTicket, schema.AnchorEndFact, "_")
// ignore errors (locations will be 0)
s, _ := strconv.Atoi(startOffset)
e, _ := strconv.Atoi(endOffset)
var err error
loc, err = norm.Location(&xpb.Location{
Kind: xpb.Location_SPAN,
Start: &xpb.Location_Point{ByteOffset: int32(s)},
End: &xpb.Location_Point{ByteOffset: int32(e)},
})
if err != nil {
return fmt.Errorf("error normalizing reference location for anchor %q: %v", ref.SourceTicket, err)
}
}
r := strings.NewReplacer(
"@source@", ref.SourceTicket,
"@target@", ref.TargetTicket,
"@edgeKind@", ref.Kind,
"@nodeKind@", nodeKind,
"@subkind@", subkind,
"@^offset@", itoa(loc.Start.ByteOffset),
"@^line@", itoa(loc.Start.LineNumber),
"@^col@", itoa(loc.Start.ColumnOffset),
"@$offset@", itoa(loc.End.ByteOffset),
"@$line@", itoa(loc.End.LineNumber),
"@$col@", itoa(loc.End.ColumnOffset),
)
if _, err := r.WriteString(out, refFormat+"\n"); err != nil {
return err
}
}
return nil
}