Beispiel #1
0
// Decorations implements part of the xrefs.Interface.
func (d *DB) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
	if d.selectText == nil {
		var err error
		d.selectText, err = d.Prepare("SELECT text, text_encoding FROM Nodes WHERE ticket = $1;")
		if err != nil {
			return nil, fmt.Errorf("error preparing selectText statement: %v", err)
		}
	}

	// TODO(schroederc): dirty buffers
	// TODO(schroederc): span locations

	fileTicket, err := kytheuri.Fix(req.Location.Ticket)
	if err != nil {
		return nil, fmt.Errorf("invalid location ticket: %v", err)
	}
	req.Location.Ticket = fileTicket

	decor := &xpb.DecorationsReply{Location: req.Location}

	r := d.selectText.QueryRow(fileTicket)
	var text []byte
	var textEncoding sql.NullString
	if err := r.Scan(&text, &textEncoding); err != nil {
		return nil, err
	}
	norm := xrefs.NewNormalizer(text)

	if req.SourceText {
		decor.SourceText = text
		decor.Encoding = textEncoding.String
	}

	if req.References {
		var err error
		decor.Reference, err = d.scanReferences(fileTicket, norm)
		if err != nil {
			return nil, err
		}

		if len(req.Filter) > 0 && len(decor.Reference) > 0 {
			var nodeTickets stringset.Set
			for _, r := range decor.Reference {
				nodeTickets.Add(r.TargetTicket)
			}

			nodes, err := d.Nodes(ctx, &gpb.NodesRequest{
				Ticket: nodeTickets.Elements(),
				Filter: req.Filter,
			})
			if err != nil {
				return nil, fmt.Errorf("error filtering nodes:%v", err)
			}
			decor.Nodes = nodes.Nodes
		}
	}

	return decor, nil
}
Beispiel #2
0
func displayTargets(edges map[string]*gpb.EdgeSet) error {
	var targets stringset.Set
	for _, es := range edges {
		for _, g := range es.Groups {
			for _, e := range g.Edge {
				targets.Add(e.TargetTicket)
			}
		}
	}

	if *displayJSON {
		return json.NewEncoder(out).Encode(targets.Elements())
	}

	for target := range targets {
		if _, err := fmt.Fprintln(out, target); err != nil {
			return err
		}
	}
	return nil
}
Beispiel #3
0
// CrossReferences implements part of the xrefs Service interface.
func (g *GraphStoreService) CrossReferences(ctx context.Context, req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
	// TODO(zarko): Callgraph integration.
	if len(req.Ticket) == 0 {
		return nil, errors.New("no cross-references requested")
	}

	requestedPageSize := int(req.PageSize)
	if requestedPageSize == 0 {
		requestedPageSize = defaultXRefPageSize
	}

	eReply, err := g.Edges(ctx, &gpb.EdgesRequest{
		Ticket:    req.Ticket,
		PageSize:  int32(requestedPageSize),
		PageToken: req.PageToken,
	})
	if err != nil {
		return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
	}

	reply := &xpb.CrossReferencesReply{
		CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),

		NextPageToken: eReply.NextPageToken,
	}
	var allRelatedNodes stringset.Set
	if len(req.Filter) > 0 {
		reply.Nodes = make(map[string]*cpb.NodeInfo)
	}

	// Cache parent files across all anchors
	files := make(map[string]*fileNode)

	var totalXRefs int
	for {
		for source, es := range eReply.EdgeSets {
			xr, ok := reply.CrossReferences[source]
			if !ok {
				xr = &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: source}
			}

			var count int
			for kind, grp := range es.Groups {
				switch {
				// TODO(schroeder): handle declarations
				case xrefs.IsDefKind(req.DefinitionKind, kind, false):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving definition anchors: %v", err)
					}
					count += len(anchors)
					xr.Definition = append(xr.Definition, anchors...)
				case xrefs.IsRefKind(req.ReferenceKind, kind):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving reference anchors: %v", err)
					}
					count += len(anchors)
					xr.Reference = append(xr.Reference, anchors...)
				case xrefs.IsDocKind(req.DocumentationKind, kind):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving documentation anchors: %v", err)
					}
					count += len(anchors)
					xr.Documentation = append(xr.Documentation, anchors...)
				case !allRelatedNodes.Empty() && !edges.IsAnchorEdge(kind):
					count += len(grp.Edge)
					for _, edge := range grp.Edge {
						xr.RelatedNode = append(xr.RelatedNode, &xpb.CrossReferencesReply_RelatedNode{
							Ticket:       edge.TargetTicket,
							RelationKind: kind,
							Ordinal:      edge.Ordinal,
						})
						allRelatedNodes.Add(edge.TargetTicket)
					}
				}
			}

			if count > 0 {
				reply.CrossReferences[xr.Ticket] = xr
				totalXRefs += count
			}
		}

		if reply.NextPageToken == "" || totalXRefs > 0 {
			break
		}

		// We need to return at least 1 xref, if there are any
		log.Println("Extra CrossReferences Edges call: ", reply.NextPageToken)
		eReply, err = g.Edges(ctx, &gpb.EdgesRequest{
			Ticket:    req.Ticket,
			PageSize:  int32(requestedPageSize),
			PageToken: reply.NextPageToken,
		})
		if err != nil {
			return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
		}
		reply.NextPageToken = eReply.NextPageToken
	}

	if !allRelatedNodes.Empty() {
		nReply, err := g.Nodes(ctx, &gpb.NodesRequest{
			Ticket: allRelatedNodes.Elements(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("error retrieving related nodes: %v", err)
		}
		for ticket, n := range nReply.Nodes {
			reply.Nodes[ticket] = n
		}
	}

	return reply, nil
}
Beispiel #4
0
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
	if len(req.DirtyBuffer) > 0 {
		return nil, errors.New("UNIMPLEMENTED: dirty buffers")
	} else if req.GetLocation() == nil {
		// TODO(schroederc): allow empty location when given dirty buffer
		return nil, errors.New("missing location")
	}

	fileVName, err := kytheuri.ToVName(req.Location.Ticket)
	if err != nil {
		return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
	}

	text, encoding, err := getSourceText(ctx, g.gs, fileVName)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve file text: %v", err)
	}
	norm := xrefs.NewNormalizer(text)

	loc, err := norm.Location(req.GetLocation())
	if err != nil {
		return nil, err
	}

	reply := &xpb.DecorationsReply{
		Location: loc,
		Nodes:    make(map[string]*cpb.NodeInfo),
	}

	// Handle DecorationsRequest.SourceText switch
	if req.SourceText {
		if loc.Kind == xpb.Location_FILE {
			reply.SourceText = text
		} else {
			reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
		}
		reply.Encoding = encoding
	}

	// Handle DecorationsRequest.References switch
	if req.References {
		// Traverse the following chain of edges:
		//   file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
		//
		// Add []anchor and []target nodes to reply.Nodes
		// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference

		patterns := xrefs.ConvertFilters(req.Filter)

		children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
			return e.EdgeKind == revChildOfEdgeKind
		})
		if err != nil {
			return nil, fmt.Errorf("failed to retrieve file children: %v", err)
		}

		var targetSet stringset.Set
		for _, edge := range children {
			anchor := edge.Target
			ticket := kytheuri.ToString(anchor)
			anchorNodeReply, err := g.Nodes(ctx, &gpb.NodesRequest{
				Ticket: []string{ticket},
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference source node: %v", err)
			} else if len(anchorNodeReply.Nodes) != 1 {
				return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
			}

			node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
			if !ok {
				return nil, fmt.Errorf("failed to find info for node %q", ticket)
			} else if string(node[facts.NodeKind]) != nodes.Anchor {
				// Skip child if it isn't an anchor node
				continue
			}

			anchorStart, err := strconv.Atoi(string(node[facts.AnchorStart]))
			if err != nil {
				log.Printf("Invalid anchor start offset %q for node %q: %v", node[facts.AnchorStart], ticket, err)
				continue
			}
			anchorEnd, err := strconv.Atoi(string(node[facts.AnchorEnd]))
			if err != nil {
				log.Printf("Invalid anchor end offset %q for node %q: %v", node[facts.AnchorEnd], ticket, err)
				continue
			}

			if loc.Kind == xpb.Location_SPAN {
				// Check if anchor fits within/around requested source text window
				if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
					continue
				} else if anchorStart > anchorEnd {
					log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
					continue
				}
			}

			targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
				return edges.IsForward(e.EdgeKind) && e.EdgeKind != edges.ChildOf
			})
			if err != nil {
				return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
			}
			if len(targets) == 0 {
				log.Printf("Anchor missing forward edges: {%+v}", anchor)
				continue
			}

			if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
				reply.Nodes[ticket] = node
			}
			for _, edge := range targets {
				targetTicket := kytheuri.ToString(edge.Target)
				targetSet.Add(targetTicket)
				reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
					SourceTicket: ticket,
					Kind:         edge.Kind,
					TargetTicket: targetTicket,
					AnchorStart:  norm.ByteOffset(int32(anchorStart)),
					AnchorEnd:    norm.ByteOffset(int32(anchorEnd)),
				})
			}
		}
		sort.Sort(bySpan(reply.Reference))

		// Only request Nodes when there are fact filters given.
		if len(req.Filter) > 0 {
			// Ensure returned nodes are not duplicated.
			for ticket := range reply.Nodes {
				targetSet.Discard(ticket)
			}

			// Batch request all Reference target nodes
			nodesReply, err := g.Nodes(ctx, &gpb.NodesRequest{
				Ticket: targetSet.Elements(),
				Filter: req.Filter,
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
			}
			for ticket, node := range nodesReply.Nodes {
				reply.Nodes[ticket] = node
			}
		}
	}

	return reply, nil
}
Beispiel #5
0
// Edges implements part of the Service interface.
func (g *GraphStoreService) Edges(ctx context.Context, req *gpb.EdgesRequest) (*gpb.EdgesReply, error) {
	if len(req.Ticket) == 0 {
		return nil, errors.New("no tickets specified")
	} else if req.PageToken != "" {
		return nil, errors.New("UNIMPLEMENTED: page_token")
	}

	patterns := xrefs.ConvertFilters(req.Filter)
	allowedKinds := stringset.New(req.Kind...)
	var targetSet stringset.Set
	reply := &gpb.EdgesReply{
		EdgeSets: make(map[string]*gpb.EdgeSet),
		Nodes:    make(map[string]*cpb.NodeInfo),
	}

	for _, ticket := range req.Ticket {
		vname, err := kytheuri.ToVName(ticket)
		if err != nil {
			return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err)
		}

		var (
			// EdgeKind -> TargetTicket -> OrdinalSet
			filteredEdges = make(map[string]map[string]map[int32]struct{})
			filteredFacts = make(map[string][]byte)
		)

		if err := g.gs.Read(ctx, &spb.ReadRequest{
			Source:   vname,
			EdgeKind: "*",
		}, func(entry *spb.Entry) error {
			edgeKind := entry.EdgeKind
			if edgeKind == "" {
				// node fact
				if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) {
					filteredFacts[entry.FactName] = entry.FactValue
				}
			} else {
				// edge
				edgeKind, ordinal, _ := edges.ParseOrdinal(edgeKind)
				if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) {
					targets, ok := filteredEdges[edgeKind]
					if !ok {
						targets = make(map[string]map[int32]struct{})
						filteredEdges[edgeKind] = targets
					}
					ticket := kytheuri.ToString(entry.Target)
					ordSet, ok := targets[ticket]
					if !ok {
						ordSet = make(map[int32]struct{})
						targets[ticket] = ordSet
					}
					ordSet[int32(ordinal)] = struct{}{}
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket)
		}

		// Only add a EdgeSet if there are targets for the requested edge kinds.
		if len(filteredEdges) > 0 {
			groups := make(map[string]*gpb.EdgeSet_Group)
			for edgeKind, targets := range filteredEdges {
				g := &gpb.EdgeSet_Group{}
				for target, ordinals := range targets {
					for ordinal := range ordinals {
						g.Edge = append(g.Edge, &gpb.EdgeSet_Group_Edge{
							TargetTicket: target,
							Ordinal:      ordinal,
						})
					}
					targetSet.Add(target)
				}
				groups[edgeKind] = g
			}
			reply.EdgeSets[ticket] = &gpb.EdgeSet{
				Groups: groups,
			}

			// In addition, only add a NodeInfo if the filters have resulting facts.
			if len(filteredFacts) > 0 {
				reply.Nodes[ticket] = &cpb.NodeInfo{
					Facts: filteredFacts,
				}
			}
		}
	}

	// Only request Nodes when there are fact filters given.
	if len(req.Filter) > 0 {
		// Eliminate redundant work by removing already requested nodes from targetSet
		for ticket := range reply.Nodes {
			targetSet.Discard(ticket)
		}

		// Batch request all leftover target nodes
		nodesReply, err := g.Nodes(ctx, &gpb.NodesRequest{
			Ticket: targetSet.Elements(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("failure getting target nodes: %v", err)
		}
		for ticket, node := range nodesReply.Nodes {
			reply.Nodes[ticket] = node
		}
	}

	return reply, nil
}
Beispiel #6
0
func (d *DB) edges(ctx context.Context, req *gpb.EdgesRequest, edgeFilter func(kind string) bool) (*gpb.EdgesReply, error) {
	tickets, err := xrefs.FixTickets(req.Ticket)
	if err != nil {
		return nil, err
	}

	pageSize := int(req.PageSize)
	if pageSize <= 0 {
		pageSize = defaultPageSize
	} else if pageSize > maxPageSize {
		pageSize = maxPageSize
	}

	var pageOffset int
	if req.PageToken != "" {
		rec, err := base64.StdEncoding.DecodeString(req.PageToken)
		if err != nil {
			return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
		}
		var t ipb.PageToken
		if err := proto.Unmarshal(rec, &t); err != nil || t.Index < 0 {
			return nil, fmt.Errorf("invalid page_token: %q", req.PageToken)
		}
		pageOffset = int(t.Index)
	}

	// Select only the edges from the given source tickets
	setQ, args := sqlSetQuery(1, tickets)
	query := fmt.Sprintf(`
SELECT * FROM AllEdges
WHERE source IN %s`, setQ)

	// Filter by edges kinds, if given
	if len(req.Kind) > 0 {
		kSetQ, kArgs := sqlSetQuery(1+len(args), req.Kind)
		query += fmt.Sprintf(`
AND kind IN %s`, kSetQ)
		args = append(args, kArgs...)
	}

	// Scan edge sets/groups in order; necessary for CrossReferences
	query += " ORDER BY source, kind, ordinal"

	// Seek to the requested page offset (req.PageToken.Index).  We don't use
	// LIMIT here because we don't yet know how many edges will be filtered by
	// edgeFilter.
	query += fmt.Sprintf(" OFFSET $%d", len(args)+1)
	args = append(args, pageOffset)

	rs, err := d.Query(query, args...)
	if err != nil {
		return nil, fmt.Errorf("error querying for edges: %v", err)
	}
	defer closeRows(rs)

	var scanned int
	// edges := map { source -> kind -> target -> ordinal set }
	edges := make(map[string]map[string]map[string]map[int32]struct{}, len(tickets))
	for count := 0; count < pageSize && rs.Next(); scanned++ {
		var source, kind, target string
		var ordinal int
		if err := rs.Scan(&source, &kind, &target, &ordinal); err != nil {
			return nil, fmt.Errorf("edges scan error: %v", err)
		}
		if edgeFilter != nil && !edgeFilter(kind) {
			continue
		}
		count++

		groups, ok := edges[source]
		if !ok {
			groups = make(map[string]map[string]map[int32]struct{})
			edges[source] = groups
		}
		targets, ok := groups[kind]
		if !ok {
			targets = make(map[string]map[int32]struct{})
			groups[kind] = targets
		}
		ordinals, ok := targets[target]
		if !ok {
			ordinals = make(map[int32]struct{})
			targets[target] = ordinals
		}
		ordinals[int32(ordinal)] = struct{}{}
	}

	reply := &gpb.EdgesReply{EdgeSets: make(map[string]*gpb.EdgeSet, len(edges))}
	var nodeTickets stringset.Set
	for src, groups := range edges {
		gs := make(map[string]*gpb.EdgeSet_Group, len(groups))
		nodeTickets.Add(src)
		for kind, targets := range groups {
			edges := make([]*gpb.EdgeSet_Group_Edge, 0, len(targets))
			for ticket, ordinals := range targets {
				for ordinal := range ordinals {
					edges = append(edges, &gpb.EdgeSet_Group_Edge{
						TargetTicket: ticket,
						Ordinal:      ordinal,
					})
				}
				nodeTickets.Add(ticket)
			}
			sort.Sort(xrefs.ByOrdinal(edges))
			gs[kind] = &gpb.EdgeSet_Group{
				Edge: edges,
			}
		}
		reply.EdgeSets[src] = &gpb.EdgeSet{
			Groups: gs,
		}
	}

	// If there is another row, there is a NextPageToken.
	if rs.Next() {
		rec, err := proto.Marshal(&ipb.PageToken{Index: int32(pageOffset + scanned)})
		if err != nil {
			return nil, fmt.Errorf("internal error: error marshalling page token: %v", err)
		}
		reply.NextPageToken = base64.StdEncoding.EncodeToString(rec)
	}

	// TODO(schroederc): faster node lookups
	if len(req.Filter) > 0 && !nodeTickets.Empty() {
		nodes, err := d.Nodes(ctx, &gpb.NodesRequest{
			Ticket: nodeTickets.Elements(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("error filtering nodes:%v", err)
		}
		reply.Nodes = nodes.Nodes
	}

	return reply, nil
}