Exemple #1
0
func edgeKindLess(kind1, kind2 string) bool {
	// General ordering:
	//   anchor edge kinds before non-anchor edge kinds
	//   forward edges before reverse edges
	//   edgeOrdering[i] (and variants) before edgeOrdering[i+1:]
	//   edge variants after root edge kind (ordered lexicographically)
	//   otherwise, order lexicographically

	if kind1 == kind2 {
		return false
	} else if a1, a2 := edges.IsAnchorEdge(kind1), edges.IsAnchorEdge(kind2); a1 != a2 {
		return a1
	} else if d1, d2 := edges.IsForward(kind1), edges.IsForward(kind2); d1 != d2 {
		return d1
	}
	kind1, kind2 = edges.Canonical(kind1), edges.Canonical(kind2)
	for _, kind := range edgeOrdering {
		if kind1 == kind {
			return true
		} else if kind2 == kind {
			return false
		} else if v1, v2 := edges.IsVariant(kind1, kind), edges.IsVariant(kind2, kind); v1 != v2 {
			return v1
		} else if v1 {
			return kind1 < kind2
		}
	}
	return kind1 < kind2
}
Exemple #2
0
func addReverseEdges(ctx context.Context, gs graphstore.Service) error {
	log.Println("Adding reverse edges")
	var (
		totalEntries int
		addedEdges   int
	)
	startTime := time.Now()
	err := gs.Scan(ctx, new(spb.ScanRequest), func(entry *spb.Entry) error {
		kind := entry.EdgeKind
		if kind != "" && edges.IsForward(kind) {
			if err := gs.Write(ctx, &spb.WriteRequest{
				Source: entry.Target,
				Update: []*spb.WriteRequest_Update{{
					Target:    entry.Source,
					EdgeKind:  edges.Mirror(kind),
					FactName:  entry.FactName,
					FactValue: entry.FactValue,
				}},
			}); err != nil {
				return fmt.Errorf("Failed to write reverse edge: %v", err)
			}
			addedEdges++
		}
		totalEntries++
		return nil
	})
	log.Printf("Wrote %d reverse edges to GraphStore (%d total entries): %v", addedEdges, totalEntries, time.Since(startTime))
	return err
}
Exemple #3
0
func filterReverses(rd stream.EntryReader) stream.EntryReader {
	return func(f func(*spb.Entry) error) error {
		return rd(func(e *spb.Entry) error {
			if graphstore.IsNodeFact(e) || edges.IsForward(e.EdgeKind) {
				return f(e)
			}
			return nil
		})
	}
}
Exemple #4
0
// ToString returns a human-readable string representation of p.
func ToString(p *ipb.Path) string {
	s := "**" + p.Pivot.NodeKind + "**"
	for _, e := range p.Edges {
		if edges.IsForward(e.Kind) {
			s += fmt.Sprintf(" -[%s]> %s", e.Kind, e.Target.NodeKind)
		} else {
			s += fmt.Sprintf(" <[%s]- %s", edges.Mirror(e.Kind), e.Target.NodeKind)
		}
	}
	return s
}
Exemple #5
0
// FromSource creates a set of *ipb.Paths for each of its edges as well as a
// single *ipb.Path with only its pivot set to be the source node.
func FromSource(src *ipb.Source) []*ipb.Path {
	var paths []*ipb.Path

	n := specialize(assemble.Node(src))
	paths = append(paths, &ipb.Path{Pivot: n})
	for kind, group := range src.EdgeGroups {
		if !edges.IsForward(kind) {
			continue
		}
		for _, tgt := range group.Edges {
			paths = append(paths, &ipb.Path{
				Pivot: &ipb.Path_Node{
					Ticket: tgt.Ticket,
				},
				Edges: []*ipb.Path_Edge{{
					Kind:    edges.Mirror(kind),
					Ordinal: int32(tgt.Ordinal),
					Target:  n,
				}},
			})
		}
	}
	return paths
}
Exemple #6
0
func (d *DB) copyEntries(entries <-chan *spb.Entry) error {
	// Start a transaction for a COPY statement per table
	nodesTx, err := d.Begin()
	if err != nil {
		return err
	}
	edgesTx, err := d.Begin()
	if err != nil {
		return err
	}

	// Create each table in their corresponding transactions to speed up COPY
	if _, err := nodesTx.Exec(createNodesTable); err != nil {
		return fmt.Errorf("error truncating Nodes table: %v", err)
	} else if _, err := edgesTx.Exec(createEdgeTable); err != nil {
		return fmt.Errorf("error truncating Edges table: %v", err)
	}

	copyNode, err := nodesTx.Prepare(pq.CopyIn(
		"nodes",
		"ticket",
		"node_kind",
		"subkind",
		"text",
		"text_encoding",
		"start_offset",
		"end_offset",
		"snippet_start",
		"snippet_end",
		"other_facts_num",
		"other_facts",
	))
	if err != nil {
		return fmt.Errorf("error preparing Nodes copy: %v", err)
	}

	copyEdge, err := edgesTx.Prepare(pq.CopyIn(
		"edges",
		"source",
		"kind",
		"target",
		"ordinal",
	))
	if err != nil {
		return fmt.Errorf("error preparing Edges copy: %v", err)
	}

	var node srvpb.Node
	var nodeKind string
	var subkind, textEncoding *string
	var text *[]byte
	var startOffset, endOffset, snippetStart, snippetEnd *int64
	for e := range entries {
		if graphstore.IsNodeFact(e) {
			ticket := kytheuri.ToString(e.Source)
			if node.Ticket != "" && node.Ticket != ticket {
				nodeTicket := node.Ticket
				node.Ticket = ""
				var rec []byte
				if len(node.Fact) > 0 {
					rec, err = proto.Marshal(&node)
					if err != nil {
						return fmt.Errorf("error marshaling facts: %v", err)
					}
				}
				if text != nil && textEncoding == nil {
					textEncoding = proto.String(facts.DefaultTextEncoding)
				}
				if _, err := copyNode.Exec(
					nodeTicket,
					nodeKind,
					subkind,
					text,
					textEncoding,
					startOffset,
					endOffset,
					snippetStart,
					snippetEnd,
					len(node.Fact),
					rec,
				); err != nil {
					return fmt.Errorf("error copying node: %v", err)
				}
				node.Fact, text = node.Fact[0:0], nil
				nodeKind = ""
				subkind, textEncoding = nil, nil
				startOffset, endOffset, snippetStart, snippetEnd = nil, nil, nil, nil
			}
			if node.Ticket == "" {
				node.Ticket = ticket
			}
			switch e.FactName {
			case facts.NodeKind:
				nodeKind = string(e.FactValue)
			case facts.Subkind:
				subkind = proto.String(string(e.FactValue))
			case facts.Text:
				text = &e.FactValue
			case facts.TextEncoding:
				textEncoding = proto.String(string(e.FactValue))
			case facts.AnchorStart:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					startOffset = proto.Int64(n)
				}
			case facts.AnchorEnd:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					endOffset = proto.Int64(n)
				}
			case facts.SnippetStart:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					snippetStart = proto.Int64(n)
				}
			case facts.SnippetEnd:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					snippetEnd = proto.Int64(n)
				}
			default:
				node.Fact = append(node.Fact, &cpb.Fact{
					Name:  e.FactName,
					Value: e.FactValue,
				})
			}
		} else if edges.IsForward(e.EdgeKind) {
			kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
			ticket := kytheuri.ToString(e.Source)
			if _, err := copyEdge.Exec(ticket, kind, kytheuri.ToString(e.Target), ordinal); err != nil {
				return fmt.Errorf("error copying edge: %v", err)
			}
		}
	}

	if _, err := copyNode.Exec(); err != nil {
		return fmt.Errorf("error flushing nodes: %v", err)
	} else if _, err := copyEdge.Exec(); err != nil {
		return fmt.Errorf("error flushing edges: %v", err)
	}

	if err := nodesTx.Commit(); err != nil {
		return fmt.Errorf("error committing Nodes transaction: %v", err)
	} else if err := edgesTx.Commit(); err != nil {
		return fmt.Errorf("error committing Edges transaction: %v", err)
	}

	return nil
}
Exemple #7
0
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
	if len(req.DirtyBuffer) > 0 {
		return nil, errors.New("UNIMPLEMENTED: dirty buffers")
	} else if req.GetLocation() == nil {
		// TODO(schroederc): allow empty location when given dirty buffer
		return nil, errors.New("missing location")
	}

	fileVName, err := kytheuri.ToVName(req.Location.Ticket)
	if err != nil {
		return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
	}

	text, encoding, err := getSourceText(ctx, g.gs, fileVName)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve file text: %v", err)
	}
	norm := xrefs.NewNormalizer(text)

	loc, err := norm.Location(req.GetLocation())
	if err != nil {
		return nil, err
	}

	reply := &xpb.DecorationsReply{
		Location: loc,
		Nodes:    make(map[string]*cpb.NodeInfo),
	}

	// Handle DecorationsRequest.SourceText switch
	if req.SourceText {
		if loc.Kind == xpb.Location_FILE {
			reply.SourceText = text
		} else {
			reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
		}
		reply.Encoding = encoding
	}

	// Handle DecorationsRequest.References switch
	if req.References {
		// Traverse the following chain of edges:
		//   file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
		//
		// Add []anchor and []target nodes to reply.Nodes
		// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference

		patterns := xrefs.ConvertFilters(req.Filter)

		children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
			return e.EdgeKind == revChildOfEdgeKind
		})
		if err != nil {
			return nil, fmt.Errorf("failed to retrieve file children: %v", err)
		}

		var targetSet stringset.Set
		for _, edge := range children {
			anchor := edge.Target
			ticket := kytheuri.ToString(anchor)
			anchorNodeReply, err := g.Nodes(ctx, &gpb.NodesRequest{
				Ticket: []string{ticket},
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference source node: %v", err)
			} else if len(anchorNodeReply.Nodes) != 1 {
				return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
			}

			node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
			if !ok {
				return nil, fmt.Errorf("failed to find info for node %q", ticket)
			} else if string(node[facts.NodeKind]) != nodes.Anchor {
				// Skip child if it isn't an anchor node
				continue
			}

			anchorStart, err := strconv.Atoi(string(node[facts.AnchorStart]))
			if err != nil {
				log.Printf("Invalid anchor start offset %q for node %q: %v", node[facts.AnchorStart], ticket, err)
				continue
			}
			anchorEnd, err := strconv.Atoi(string(node[facts.AnchorEnd]))
			if err != nil {
				log.Printf("Invalid anchor end offset %q for node %q: %v", node[facts.AnchorEnd], ticket, err)
				continue
			}

			if loc.Kind == xpb.Location_SPAN {
				// Check if anchor fits within/around requested source text window
				if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
					continue
				} else if anchorStart > anchorEnd {
					log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
					continue
				}
			}

			targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
				return edges.IsForward(e.EdgeKind) && e.EdgeKind != edges.ChildOf
			})
			if err != nil {
				return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
			}
			if len(targets) == 0 {
				log.Printf("Anchor missing forward edges: {%+v}", anchor)
				continue
			}

			if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
				reply.Nodes[ticket] = node
			}
			for _, edge := range targets {
				targetTicket := kytheuri.ToString(edge.Target)
				targetSet.Add(targetTicket)
				reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
					SourceTicket: ticket,
					Kind:         edge.Kind,
					TargetTicket: targetTicket,
					AnchorStart:  norm.ByteOffset(int32(anchorStart)),
					AnchorEnd:    norm.ByteOffset(int32(anchorEnd)),
				})
			}
		}
		sort.Sort(bySpan(reply.Reference))

		// Only request Nodes when there are fact filters given.
		if len(req.Filter) > 0 {
			// Ensure returned nodes are not duplicated.
			for ticket := range reply.Nodes {
				targetSet.Discard(ticket)
			}

			// Batch request all Reference target nodes
			nodesReply, err := g.Nodes(ctx, &gpb.NodesRequest{
				Ticket: targetSet.Elements(),
				Filter: req.Filter,
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
			}
			for ticket, node := range nodesReply.Nodes {
				reply.Nodes[ticket] = node
			}
		}
	}

	return reply, nil
}