// Edges implements part of the xrefs Service interface.
func (t *tableImpl) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
allowedKinds := stringset.New(req.Kind...)
return t.edges(ctx, edgesRequest{
Tickets: tickets,
Filters: req.Filter,
Kinds: func(kind string) bool {
return len(allowedKinds) == 0 || allowedKinds.Contains(kind)
},
PageSize: int(req.PageSize),
PageToken: req.PageToken,
})
}
// Nodes implements part of the xrefs Service interface.
func (t *tableImpl) Nodes(ctx context.Context, req *xpb.NodesRequest) (*xpb.NodesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
rs, err := t.pagedEdgeSets(ctx, tickets)
if err != nil {
return nil, err
}
defer func() {
// drain channel in case of errors
for _ = range rs {
}
}()
reply := &xpb.NodesReply{Nodes: make(map[string]*xpb.NodeInfo, len(req.Ticket))}
patterns := xrefs.ConvertFilters(req.Filter)
for r := range rs {
if r.Err == table.ErrNoSuchKey {
continue
} else if r.Err != nil {
return nil, r.Err
}
node := r.PagedEdgeSet.Source
ni := &xpb.NodeInfo{Facts: make(map[string][]byte, len(node.Fact))}
for _, f := range node.Fact {
if len(patterns) == 0 || xrefs.MatchesAny(f.Name, patterns) {
ni.Facts[f.Name] = f.Value
}
}
if len(ni.Facts) > 0 {
reply.Nodes[node.Ticket] = ni
}
}
return reply, nil
}
// CrossReferences implements part of the xrefs.Service interface.
func (t *tableImpl) CrossReferences(ctx context.Context, req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
stats := refStats{
max: int(req.PageSize),
}
if stats.max < 0 {
return nil, fmt.Errorf("invalid page_size: %d", req.PageSize)
} else if stats.max == 0 {
stats.max = defaultPageSize
} else if stats.max > maxPageSize {
stats.max = maxPageSize
}
var edgesPageToken string
if req.PageToken != "" {
rec, err := base64.StdEncoding.DecodeString(req.PageToken)
if err != nil {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
var t ipb.PageToken
if err := proto.Unmarshal(rec, &t); err != nil || t.Index < 0 {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
stats.skip = int(t.Index)
edgesPageToken = t.SecondaryToken
}
pageToken := stats.skip
reply := &xpb.CrossReferencesReply{
CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet, len(req.Ticket)),
Nodes: make(map[string]*xpb.NodeInfo, len(req.Ticket)),
Total: &xpb.CrossReferencesReply_Total{},
}
var nextToken *ipb.PageToken
wantMoreCrossRefs := edgesPageToken == "" &&
(req.DefinitionKind != xpb.CrossReferencesRequest_NO_DEFINITIONS ||
req.DeclarationKind != xpb.CrossReferencesRequest_NO_DECLARATIONS ||
req.ReferenceKind != xpb.CrossReferencesRequest_NO_REFERENCES ||
req.DocumentationKind != xpb.CrossReferencesRequest_NO_DOCUMENTATION ||
req.CallerKind != xpb.CrossReferencesRequest_NO_CALLERS)
for _, ticket := range tickets {
// TODO(schroederc): retrieve PagedCrossReferences in parallel
cr, err := t.crossReferences(ctx, ticket)
if err == table.ErrNoSuchKey {
log.Println("Missing CrossReferences:", ticket)
continue
} else if err != nil {
return nil, fmt.Errorf("error looking up cross-references for ticket %q: %v", ticket, err)
}
crs := &xpb.CrossReferencesReply_CrossReferenceSet{
Ticket: ticket,
}
if req.ExperimentalSignatures {
crs.DisplayName, err = xrefs.SlowSignature(ctx, t, ticket)
if err != nil {
log.Printf("WARNING: error looking up signature for ticket %q: %v", ticket, err)
}
}
for _, grp := range cr.Group {
switch {
case xrefs.IsDefKind(req.DefinitionKind, grp.Kind, cr.Incomplete):
reply.Total.Definitions += int64(len(grp.Anchor))
if wantMoreCrossRefs {
stats.addAnchors(&crs.Definition, grp.Anchor, req.AnchorText)
}
case xrefs.IsDeclKind(req.DeclarationKind, grp.Kind, cr.Incomplete):
reply.Total.Declarations += int64(len(grp.Anchor))
if wantMoreCrossRefs {
stats.addAnchors(&crs.Declaration, grp.Anchor, req.AnchorText)
}
case xrefs.IsDocKind(req.DocumentationKind, grp.Kind):
reply.Total.Documentation += int64(len(grp.Anchor))
if wantMoreCrossRefs {
stats.addAnchors(&crs.Documentation, grp.Anchor, req.AnchorText)
}
case xrefs.IsRefKind(req.ReferenceKind, grp.Kind):
reply.Total.References += int64(len(grp.Anchor))
if wantMoreCrossRefs {
stats.addAnchors(&crs.Reference, grp.Anchor, req.AnchorText)
}
}
}
if wantMoreCrossRefs && req.CallerKind != xpb.CrossReferencesRequest_NO_CALLERS {
anchors, err := xrefs.SlowCallersForCrossReferences(ctx, t, req.CallerKind == xpb.CrossReferencesRequest_OVERRIDE_CALLERS, req.ExperimentalSignatures, ticket)
if err != nil {
return nil, fmt.Errorf("error in SlowCallersForCrossReferences: %v", err)
}
reply.Total.Callers += int64(len(anchors))
stats.addRelatedAnchors(&crs.Caller, anchors, req.AnchorText)
}
for _, idx := range cr.PageIndex {
switch {
case xrefs.IsDefKind(req.DefinitionKind, idx.Kind, cr.Incomplete):
reply.Total.Definitions += int64(idx.Count)
if wantMoreCrossRefs && !stats.skipPage(idx) {
p, err := t.crossReferencesPage(ctx, idx.PageKey)
if err != nil {
return nil, fmt.Errorf("internal error: error retrieving cross-references page: %v", idx.PageKey)
}
stats.addAnchors(&crs.Definition, p.Group.Anchor, req.AnchorText)
}
case xrefs.IsDeclKind(req.DeclarationKind, idx.Kind, cr.Incomplete):
reply.Total.Declarations += int64(idx.Count)
if wantMoreCrossRefs && !stats.skipPage(idx) {
p, err := t.crossReferencesPage(ctx, idx.PageKey)
if err != nil {
return nil, fmt.Errorf("internal error: error retrieving cross-references page: %v", idx.PageKey)
}
stats.addAnchors(&crs.Declaration, p.Group.Anchor, req.AnchorText)
}
case xrefs.IsDocKind(req.DocumentationKind, idx.Kind):
reply.Total.Documentation += int64(idx.Count)
if wantMoreCrossRefs && !stats.skipPage(idx) {
p, err := t.crossReferencesPage(ctx, idx.PageKey)
if err != nil {
return nil, fmt.Errorf("internal error: error retrieving cross-references page: %v", idx.PageKey)
}
stats.addAnchors(&crs.Documentation, p.Group.Anchor, req.AnchorText)
}
case xrefs.IsRefKind(req.ReferenceKind, idx.Kind):
reply.Total.References += int64(idx.Count)
if wantMoreCrossRefs && !stats.skipPage(idx) {
p, err := t.crossReferencesPage(ctx, idx.PageKey)
if err != nil {
return nil, fmt.Errorf("internal error: error retrieving cross-references page: %v", idx.PageKey)
}
stats.addAnchors(&crs.Reference, p.Group.Anchor, req.AnchorText)
}
}
}
if len(crs.Declaration) > 0 || len(crs.Definition) > 0 || len(crs.Reference) > 0 || len(crs.Documentation) > 0 || len(crs.Caller) > 0 {
reply.CrossReferences[crs.Ticket] = crs
}
}
if pageToken+stats.total != sumTotalCrossRefs(reply.Total) && stats.total != 0 {
nextToken = &ipb.PageToken{Index: int32(pageToken + stats.total)}
}
if len(req.Filter) > 0 {
er, err := t.edges(ctx, edgesRequest{
Tickets: tickets,
Filters: req.Filter,
Kinds: func(kind string) bool { return !schema.IsAnchorEdge(kind) },
PageToken: edgesPageToken,
TotalOnly: (stats.max <= stats.total),
PageSize: stats.max - stats.total,
})
if err != nil {
return nil, fmt.Errorf("error getting related nodes: %v", err)
}
reply.Total.RelatedNodesByRelation = er.TotalEdgesByKind
for ticket, es := range er.EdgeSets {
nodes := stringset.New()
crs, ok := reply.CrossReferences[ticket]
if !ok {
crs = &xpb.CrossReferencesReply_CrossReferenceSet{
Ticket: ticket,
}
}
for kind, g := range es.Groups {
for _, edge := range g.Edge {
nodes.Add(edge.TargetTicket)
crs.RelatedNode = append(crs.RelatedNode, &xpb.CrossReferencesReply_RelatedNode{
RelationKind: kind,
Ticket: edge.TargetTicket,
Ordinal: edge.Ordinal,
})
}
}
if len(nodes) > 0 {
for ticket, n := range er.Nodes {
if nodes.Contains(ticket) {
reply.Nodes[ticket] = n
}
}
}
if !ok && len(crs.RelatedNode) > 0 {
reply.CrossReferences[ticket] = crs
}
}
if er.NextPageToken != "" {
nextToken = &ipb.PageToken{SecondaryToken: er.NextPageToken}
}
}
if nextToken != nil {
rec, err := proto.Marshal(nextToken)
if err != nil {
return nil, fmt.Errorf("internal error: error marshalling page token: %v", err)
}
reply.NextPageToken = base64.StdEncoding.EncodeToString(rec)
}
if req.NodeDefinitions {
nodeTickets := make([]string, 0, len(reply.Nodes))
for ticket := range reply.Nodes {
nodeTickets = append(nodeTickets, ticket)
}
// TODO(schroederc): cache this in the serving data
defs, err := xrefs.SlowDefinitions(t, ctx, nodeTickets)
if err != nil {
return nil, fmt.Errorf("error retrieving node definitions: %v", err)
}
reply.DefinitionLocations = make(map[string]*xpb.Anchor, len(defs))
for ticket, def := range defs {
node, ok := reply.Nodes[ticket]
if !ok {
panic(fmt.Sprintf("extra definition returned for unknown node %q: %v", ticket, def))
}
node.Definition = def.Ticket
if _, ok := reply.DefinitionLocations[def.Ticket]; !ok {
reply.DefinitionLocations[def.Ticket] = def
}
}
}
return reply, nil
}
// Nodes implements part of the xrefs.Interface.
func (d *DB) Nodes(ctx context.Context, req *xpb.NodesRequest) (*xpb.NodesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
setQ, args := sqlSetQuery(1, tickets)
rs, err := d.Query(fmt.Sprintf("SELECT * FROM Nodes WHERE ticket IN %s;", setQ), args...)
if err != nil {
return nil, fmt.Errorf("error querying for nodes: %v", err)
}
defer closeRows(rs)
var reply xpb.NodesReply
for rs.Next() {
var ticket, nodeKind string
var subkind, textEncoding sql.NullString
var text, otherFacts []byte
var startOffset, endOffset, snippetStart, snippetEnd sql.NullInt64
var otherFactsNum int
if err := rs.Scan(&ticket, &nodeKind, &subkind, &text, &textEncoding, &startOffset, &endOffset, &snippetStart, &snippetEnd, &otherFactsNum, &otherFacts); err != nil {
return nil, fmt.Errorf("error scanning nodes: %v", err)
}
n := new(xpb.NodeInfo)
if otherFactsNum > 0 {
if err := proto.Unmarshal(otherFacts, n); err != nil {
return nil, fmt.Errorf("unexpected node internal format: %v", err)
}
}
if nodeKind != "" {
n.Facts[ticket] = []byte(nodeKind)
}
if subkind.Valid {
n.Facts[ticket] = []byte(subkind.String)
}
if text != nil { // TODO(schroederc): NULL text
n.Facts[ticket] = text
}
if textEncoding.Valid {
n.Facts[ticket] = []byte(textEncoding.String)
}
if startOffset.Valid {
n.Facts[ticket] = []byte(strconv.FormatInt(startOffset.Int64, 10))
}
if endOffset.Valid {
n.Facts[ticket] = []byte(strconv.FormatInt(endOffset.Int64, 10))
}
if snippetStart.Valid {
n.Facts[ticket] = []byte(strconv.FormatInt(snippetStart.Int64, 10))
}
if snippetEnd.Valid {
n.Facts[ticket] = []byte(strconv.FormatInt(snippetEnd.Int64, 10))
}
if len(req.Filter) > 0 {
patterns := xrefs.ConvertFilters(req.Filter)
matched := make(map[string][]byte, len(n.Facts))
for name, value := range n.Facts {
if xrefs.MatchesAny(name, patterns) {
matched[name] = value
}
}
n.Facts = matched
}
if len(n.Facts) > 0 {
reply.Nodes[ticket] = n
}
}
return &reply, nil
}
// CrossReferences implements part of the xrefs.Interface.
func (d *DB) CrossReferences(ctx context.Context, req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
pageSize := int(req.PageSize)
if pageSize <= 0 {
pageSize = defaultPageSize
} else if pageSize > maxPageSize {
pageSize = maxPageSize
}
var pageOffset int
var edgesToken string
if req.PageToken != "" {
rec, err := base64.StdEncoding.DecodeString(req.PageToken)
if err != nil {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
var t ipb.PageToken
if err := proto.Unmarshal(rec, &t); err != nil || t.Index < 0 {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
pageOffset = int(t.Index)
edgesToken = t.SecondaryToken
}
reply := &xpb.CrossReferencesReply{
CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),
Nodes: make(map[string]*xpb.NodeInfo),
}
setQ, ticketArgs := sqlSetQuery(1, tickets)
var count int
if edgesToken == "" {
args := append(ticketArgs, pageSize+1, pageOffset) // +1 to check for next page
rs, err := d.Query(fmt.Sprintf("SELECT ticket, kind, proto FROM CrossReferences WHERE ticket IN %s ORDER BY ticket LIMIT $%d OFFSET $%d;", setQ, len(tickets)+1, len(tickets)+2), args...)
if err != nil {
return nil, err
}
defer closeRows(rs)
var xrs *xpb.CrossReferencesReply_CrossReferenceSet
for rs.Next() {
count++
if count > pageSize {
continue
}
var ticket, kind string
var rec []byte
if err := rs.Scan(&ticket, &kind, &rec); err != nil {
return nil, err
}
if xrs != nil && xrs.Ticket != ticket {
if len(xrs.Definition) > 0 || len(xrs.Documentation) > 0 || len(xrs.Reference) > 0 || len(xrs.RelatedNode) > 0 {
reply.CrossReferences[xrs.Ticket] = xrs
}
xrs = nil
}
if xrs == nil {
xrs = &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: ticket}
}
switch {
// TODO(schroederc): handle declarations
case xrefs.IsDefKind(req.DefinitionKind, kind, false):
xrs.Definition, err = addRelatedAnchor(xrs.Definition, rec, req.AnchorText)
if err != nil {
return nil, err
}
case xrefs.IsDocKind(req.DocumentationKind, kind):
xrs.Documentation, err = addRelatedAnchor(xrs.Documentation, rec, req.AnchorText)
if err != nil {
return nil, err
}
case xrefs.IsRefKind(req.ReferenceKind, kind):
xrs.Reference, err = addRelatedAnchor(xrs.Reference, rec, req.AnchorText)
if err != nil {
return nil, err
}
}
}
if xrs != nil && (len(xrs.Definition) > 0 || len(xrs.Documentation) > 0 || len(xrs.Reference) > 0 || len(xrs.RelatedNode) > 0) {
reply.CrossReferences[xrs.Ticket] = xrs
}
if count > pageSize {
rec, err := proto.Marshal(&ipb.PageToken{Index: int32(pageOffset + pageSize)})
if err != nil {
return nil, fmt.Errorf("internal error: error marshalling page token: %v", err)
}
reply.NextPageToken = base64.StdEncoding.EncodeToString(rec)
}
}
if len(req.Filter) > 0 && count <= pageSize {
// TODO(schroederc): consolidate w/ LevelDB implementation
er, err := d.edges(ctx, &xpb.EdgesRequest{
Ticket: tickets,
Filter: req.Filter,
PageSize: int32(pageSize - count),
PageToken: edgesToken,
}, func(kind string) bool {
return !schema.IsAnchorEdge(kind)
})
if err != nil {
return nil, fmt.Errorf("error getting related nodes: %v", err)
}
for ticket, es := range er.EdgeSets {
nodes := stringset.New()
crs, ok := reply.CrossReferences[ticket]
if !ok {
crs = &xpb.CrossReferencesReply_CrossReferenceSet{
Ticket: ticket,
}
}
for kind, g := range es.Groups {
if !schema.IsAnchorEdge(kind) {
for _, edge := range g.Edge {
nodes.Add(edge.TargetTicket)
crs.RelatedNode = append(crs.RelatedNode, &xpb.CrossReferencesReply_RelatedNode{
RelationKind: kind,
Ticket: edge.TargetTicket,
Ordinal: edge.Ordinal,
})
}
}
}
if len(nodes) > 0 {
for ticket, n := range er.Nodes {
if nodes.Contains(ticket) {
reply.Nodes[ticket] = n
}
}
}
if !ok && len(crs.RelatedNode) > 0 {
reply.CrossReferences[ticket] = crs
}
}
if er.NextPageToken != "" {
rec, err := proto.Marshal(&ipb.PageToken{SecondaryToken: er.NextPageToken})
if err != nil {
return nil, fmt.Errorf("internal error: error marshalling page token: %v", err)
}
reply.NextPageToken = base64.StdEncoding.EncodeToString(rec)
}
}
return reply, nil
}
func (d *DB) edges(ctx context.Context, req *xpb.EdgesRequest, edgeFilter func(kind string) bool) (*xpb.EdgesReply, error) {
tickets, err := xrefs.FixTickets(req.Ticket)
if err != nil {
return nil, err
}
pageSize := int(req.PageSize)
if pageSize <= 0 {
pageSize = defaultPageSize
} else if pageSize > maxPageSize {
pageSize = maxPageSize
}
var pageOffset int
if req.PageToken != "" {
rec, err := base64.StdEncoding.DecodeString(req.PageToken)
if err != nil {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
var t ipb.PageToken
if err := proto.Unmarshal(rec, &t); err != nil || t.Index < 0 {
return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
}
pageOffset = int(t.Index)
}
// Select only the edges from the given source tickets
setQ, args := sqlSetQuery(1, tickets)
query := fmt.Sprintf(`
SELECT * FROM AllEdges
WHERE source IN %s`, setQ)
// Filter by edges kinds, if given
if len(req.Kind) > 0 {
kSetQ, kArgs := sqlSetQuery(1+len(args), req.Kind)
query += fmt.Sprintf(`
AND kind IN %s`, kSetQ)
args = append(args, kArgs...)
}
// Scan edge sets/groups in order; necessary for CrossReferences
query += " ORDER BY source, kind, ordinal"
// Seek to the requested page offset (req.PageToken.Index). We don't use
// LIMIT here because we don't yet know how many edges will be filtered by
// edgeFilter.
query += fmt.Sprintf(" OFFSET $%d", len(args)+1)
args = append(args, pageOffset)
rs, err := d.Query(query, args...)
if err != nil {
return nil, fmt.Errorf("error querying for edges: %v", err)
}
defer closeRows(rs)
var scanned int
// edges := map { source -> kind -> target -> ordinal set }
edges := make(map[string]map[string]map[string]map[int32]struct{}, len(tickets))
for count := 0; count < pageSize && rs.Next(); scanned++ {
var source, kind, target string
var ordinal int
if err := rs.Scan(&source, &kind, &target, &ordinal); err != nil {
return nil, fmt.Errorf("edges scan error: %v", err)
}
if edgeFilter != nil && !edgeFilter(kind) {
continue
}
count++
groups, ok := edges[source]
if !ok {
groups = make(map[string]map[string]map[int32]struct{})
edges[source] = groups
}
targets, ok := groups[kind]
if !ok {
targets = make(map[string]map[int32]struct{})
groups[kind] = targets
}
ordinals, ok := targets[target]
if !ok {
ordinals = make(map[int32]struct{})
targets[target] = ordinals
}
ordinals[int32(ordinal)] = struct{}{}
}
reply := &xpb.EdgesReply{EdgeSets: make(map[string]*xpb.EdgeSet, len(edges))}
nodeTickets := stringset.New()
for src, groups := range edges {
gs := make(map[string]*xpb.EdgeSet_Group, len(groups))
nodeTickets.Add(src)
for kind, targets := range groups {
edges := make([]*xpb.EdgeSet_Group_Edge, 0, len(targets))
for ticket, ordinals := range targets {
for ordinal := range ordinals {
edges = append(edges, &xpb.EdgeSet_Group_Edge{
TargetTicket: ticket,
Ordinal: ordinal,
})
}
nodeTickets.Add(ticket)
}
sort.Sort(xrefs.ByOrdinal(edges))
gs[kind] = &xpb.EdgeSet_Group{
Edge: edges,
}
}
reply.EdgeSets[src] = &xpb.EdgeSet{
Groups: gs,
}
}
// If there is another row, there is a NextPageToken.
if rs.Next() {
rec, err := proto.Marshal(&ipb.PageToken{Index: int32(pageOffset + scanned)})
if err != nil {
return nil, fmt.Errorf("internal error: error marshalling page token: %v", err)
}
reply.NextPageToken = base64.StdEncoding.EncodeToString(rec)
}
// TODO(schroederc): faster node lookups
if len(req.Filter) > 0 && len(nodeTickets) > 0 {
nodes, err := d.Nodes(ctx, &xpb.NodesRequest{
Ticket: nodeTickets.Slice(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("error filtering nodes:%v", err)
}
reply.Nodes = nodes.Nodes
}
return reply, nil
}
