func edgeKindLess(kind1, kind2 string) bool {
// General ordering:
// anchor edge kinds before non-anchor edge kinds
// forward edges before reverse edges
// edgeOrdering[i] (and variants) before edgeOrdering[i+1:]
// edge variants after root edge kind (ordered lexicographically)
// otherwise, order lexicographically
if kind1 == kind2 {
return false
} else if a1, a2 := edges.IsAnchorEdge(kind1), edges.IsAnchorEdge(kind2); a1 != a2 {
return a1
} else if d1, d2 := edges.IsForward(kind1), edges.IsForward(kind2); d1 != d2 {
return d1
}
kind1, kind2 = edges.Canonical(kind1), edges.Canonical(kind2)
for _, kind := range edgeOrdering {
if kind1 == kind {
return true
} else if kind2 == kind {
return false
} else if v1, v2 := edges.IsVariant(kind1, kind), edges.IsVariant(kind2, kind); v1 != v2 {
return v1
} else if v1 {
return kind1 < kind2
}
}
return kind1 < kind2
}
func addReverseEdges(ctx context.Context, gs graphstore.Service) error {
log.Println("Adding reverse edges")
var (
totalEntries int
addedEdges int
)
startTime := time.Now()
err := gs.Scan(ctx, new(spb.ScanRequest), func(entry *spb.Entry) error {
kind := entry.EdgeKind
if kind != "" && edges.IsForward(kind) {
if err := gs.Write(ctx, &spb.WriteRequest{
Source: entry.Target,
Update: []*spb.WriteRequest_Update{{
Target: entry.Source,
EdgeKind: edges.Mirror(kind),
FactName: entry.FactName,
FactValue: entry.FactValue,
}},
}); err != nil {
return fmt.Errorf("Failed to write reverse edge: %v", err)
}
addedEdges++
}
totalEntries++
return nil
})
log.Printf("Wrote %d reverse edges to GraphStore (%d total entries): %v", addedEdges, totalEntries, time.Since(startTime))
return err
}
func filterReverses(rd stream.EntryReader) stream.EntryReader {
return func(f func(*spb.Entry) error) error {
return rd(func(e *spb.Entry) error {
if graphstore.IsNodeFact(e) || edges.IsForward(e.EdgeKind) {
return f(e)
}
return nil
})
}
}
// ToString returns a human-readable string representation of p.
func ToString(p *ipb.Path) string {
s := "**" + p.Pivot.NodeKind + "**"
for _, e := range p.Edges {
if edges.IsForward(e.Kind) {
s += fmt.Sprintf(" -[%s]> %s", e.Kind, e.Target.NodeKind)
} else {
s += fmt.Sprintf(" <[%s]- %s", edges.Mirror(e.Kind), e.Target.NodeKind)
}
}
return s
}
// FromSource creates a set of *ipb.Paths for each of its edges as well as a
// single *ipb.Path with only its pivot set to be the source node.
func FromSource(src *ipb.Source) []*ipb.Path {
var paths []*ipb.Path
n := specialize(assemble.Node(src))
paths = append(paths, &ipb.Path{Pivot: n})
for kind, group := range src.EdgeGroups {
if !edges.IsForward(kind) {
continue
}
for _, tgt := range group.Edges {
paths = append(paths, &ipb.Path{
Pivot: &ipb.Path_Node{
Ticket: tgt.Ticket,
},
Edges: []*ipb.Path_Edge{{
Kind: edges.Mirror(kind),
Ordinal: int32(tgt.Ordinal),
Target: n,
}},
})
}
}
return paths
}
func (d *DB) copyEntries(entries <-chan *spb.Entry) error {
// Start a transaction for a COPY statement per table
nodesTx, err := d.Begin()
if err != nil {
return err
}
edgesTx, err := d.Begin()
if err != nil {
return err
}
// Create each table in their corresponding transactions to speed up COPY
if _, err := nodesTx.Exec(createNodesTable); err != nil {
return fmt.Errorf("error truncating Nodes table: %v", err)
} else if _, err := edgesTx.Exec(createEdgeTable); err != nil {
return fmt.Errorf("error truncating Edges table: %v", err)
}
copyNode, err := nodesTx.Prepare(pq.CopyIn(
"nodes",
"ticket",
"node_kind",
"subkind",
"text",
"text_encoding",
"start_offset",
"end_offset",
"snippet_start",
"snippet_end",
"other_facts_num",
"other_facts",
))
if err != nil {
return fmt.Errorf("error preparing Nodes copy: %v", err)
}
copyEdge, err := edgesTx.Prepare(pq.CopyIn(
"edges",
"source",
"kind",
"target",
"ordinal",
))
if err != nil {
return fmt.Errorf("error preparing Edges copy: %v", err)
}
var node srvpb.Node
var nodeKind string
var subkind, textEncoding *string
var text *[]byte
var startOffset, endOffset, snippetStart, snippetEnd *int64
for e := range entries {
if graphstore.IsNodeFact(e) {
ticket := kytheuri.ToString(e.Source)
if node.Ticket != "" && node.Ticket != ticket {
nodeTicket := node.Ticket
node.Ticket = ""
var rec []byte
if len(node.Fact) > 0 {
rec, err = proto.Marshal(&node)
if err != nil {
return fmt.Errorf("error marshaling facts: %v", err)
}
}
if text != nil && textEncoding == nil {
textEncoding = proto.String(facts.DefaultTextEncoding)
}
if _, err := copyNode.Exec(
nodeTicket,
nodeKind,
subkind,
text,
textEncoding,
startOffset,
endOffset,
snippetStart,
snippetEnd,
len(node.Fact),
rec,
); err != nil {
return fmt.Errorf("error copying node: %v", err)
}
node.Fact, text = node.Fact[0:0], nil
nodeKind = ""
subkind, textEncoding = nil, nil
startOffset, endOffset, snippetStart, snippetEnd = nil, nil, nil, nil
}
if node.Ticket == "" {
node.Ticket = ticket
}
switch e.FactName {
case facts.NodeKind:
nodeKind = string(e.FactValue)
case facts.Subkind:
subkind = proto.String(string(e.FactValue))
case facts.Text:
text = &e.FactValue
case facts.TextEncoding:
textEncoding = proto.String(string(e.FactValue))
case facts.AnchorStart:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
startOffset = proto.Int64(n)
}
case facts.AnchorEnd:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
endOffset = proto.Int64(n)
}
case facts.SnippetStart:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
snippetStart = proto.Int64(n)
}
case facts.SnippetEnd:
n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
if err == nil {
snippetEnd = proto.Int64(n)
}
default:
node.Fact = append(node.Fact, &cpb.Fact{
Name: e.FactName,
Value: e.FactValue,
})
}
} else if edges.IsForward(e.EdgeKind) {
kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
ticket := kytheuri.ToString(e.Source)
if _, err := copyEdge.Exec(ticket, kind, kytheuri.ToString(e.Target), ordinal); err != nil {
return fmt.Errorf("error copying edge: %v", err)
}
}
}
if _, err := copyNode.Exec(); err != nil {
return fmt.Errorf("error flushing nodes: %v", err)
} else if _, err := copyEdge.Exec(); err != nil {
return fmt.Errorf("error flushing edges: %v", err)
}
if err := nodesTx.Commit(); err != nil {
return fmt.Errorf("error committing Nodes transaction: %v", err)
} else if err := edgesTx.Commit(); err != nil {
return fmt.Errorf("error committing Edges transaction: %v", err)
}
return nil
}
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
if len(req.DirtyBuffer) > 0 {
return nil, errors.New("UNIMPLEMENTED: dirty buffers")
} else if req.GetLocation() == nil {
// TODO(schroederc): allow empty location when given dirty buffer
return nil, errors.New("missing location")
}
fileVName, err := kytheuri.ToVName(req.Location.Ticket)
if err != nil {
return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
}
text, encoding, err := getSourceText(ctx, g.gs, fileVName)
if err != nil {
return nil, fmt.Errorf("failed to retrieve file text: %v", err)
}
norm := xrefs.NewNormalizer(text)
loc, err := norm.Location(req.GetLocation())
if err != nil {
return nil, err
}
reply := &xpb.DecorationsReply{
Location: loc,
Nodes: make(map[string]*cpb.NodeInfo),
}
// Handle DecorationsRequest.SourceText switch
if req.SourceText {
if loc.Kind == xpb.Location_FILE {
reply.SourceText = text
} else {
reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
}
reply.Encoding = encoding
}
// Handle DecorationsRequest.References switch
if req.References {
// Traverse the following chain of edges:
// file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
//
// Add []anchor and []target nodes to reply.Nodes
// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference
patterns := xrefs.ConvertFilters(req.Filter)
children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
return e.EdgeKind == revChildOfEdgeKind
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve file children: %v", err)
}
var targetSet stringset.Set
for _, edge := range children {
anchor := edge.Target
ticket := kytheuri.ToString(anchor)
anchorNodeReply, err := g.Nodes(ctx, &gpb.NodesRequest{
Ticket: []string{ticket},
})
if err != nil {
return nil, fmt.Errorf("failure getting reference source node: %v", err)
} else if len(anchorNodeReply.Nodes) != 1 {
return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
}
node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
if !ok {
return nil, fmt.Errorf("failed to find info for node %q", ticket)
} else if string(node[facts.NodeKind]) != nodes.Anchor {
// Skip child if it isn't an anchor node
continue
}
anchorStart, err := strconv.Atoi(string(node[facts.AnchorStart]))
if err != nil {
log.Printf("Invalid anchor start offset %q for node %q: %v", node[facts.AnchorStart], ticket, err)
continue
}
anchorEnd, err := strconv.Atoi(string(node[facts.AnchorEnd]))
if err != nil {
log.Printf("Invalid anchor end offset %q for node %q: %v", node[facts.AnchorEnd], ticket, err)
continue
}
if loc.Kind == xpb.Location_SPAN {
// Check if anchor fits within/around requested source text window
if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
continue
} else if anchorStart > anchorEnd {
log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
continue
}
}
targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
return edges.IsForward(e.EdgeKind) && e.EdgeKind != edges.ChildOf
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
}
if len(targets) == 0 {
log.Printf("Anchor missing forward edges: {%+v}", anchor)
continue
}
if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
reply.Nodes[ticket] = node
}
for _, edge := range targets {
targetTicket := kytheuri.ToString(edge.Target)
targetSet.Add(targetTicket)
reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
SourceTicket: ticket,
Kind: edge.Kind,
TargetTicket: targetTicket,
AnchorStart: norm.ByteOffset(int32(anchorStart)),
AnchorEnd: norm.ByteOffset(int32(anchorEnd)),
})
}
}
sort.Sort(bySpan(reply.Reference))
// Only request Nodes when there are fact filters given.
if len(req.Filter) > 0 {
// Ensure returned nodes are not duplicated.
for ticket := range reply.Nodes {
targetSet.Discard(ticket)
}
// Batch request all Reference target nodes
nodesReply, err := g.Nodes(ctx, &gpb.NodesRequest{
Ticket: targetSet.Elements(),
Filter: req.Filter,
})
if err != nil {
return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
}
for ticket, node := range nodesReply.Nodes {
reply.Nodes[ticket] = node
}
}
}
return reply, nil
}