コード例 #1
0
ファイル: pipeline.go プロジェクト: kidaa/kythe
func writeEdges(ctx context.Context, t table.Proto, edges <-chan *spb.Entry, maxEdgePageSize int) error {
	defer drainEntries(edges) // ensure channel is drained on errors

	temp, err := tempTable("edge.groups")
	if err != nil {
		return fmt.Errorf("failed to create temporary table: %v", err)
	}
	edgeGroups := &table.KVProto{temp}
	defer func() {
		if err := edgeGroups.Close(ctx); err != nil {
			log.Println("Error closing edge groups table: %v", err)
		}
	}()

	log.Println("Writing temporary edges table")

	var (
		src     *spb.VName
		kind    string
		targets stringset.Set
	)
	for e := range edges {
		if src != nil && (!compare.VNamesEqual(e.Source, src) || kind != e.EdgeKind) {
			if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
				return err
			}
			src = nil
		}
		if src == nil {
			src = e.Source
			kind = e.EdgeKind
			targets = stringset.New()
		}
		targets.Add(kytheuri.ToString(e.Target))
	}
	if src != nil {
		if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
			return err
		}
	}

	return writeEdgePages(ctx, t, edgeGroups, maxEdgePageSize)
}
コード例 #2
0
ファイル: xrefs.go プロジェクト: bzz/kythe
// CrossReferences implements part of the xrefs Service interface.
func (g *GraphStoreService) CrossReferences(ctx context.Context, req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
	if len(req.Ticket) == 0 {
		return nil, errors.New("no cross-references requested")
	}

	requestedPageSize := int(req.PageSize)
	if requestedPageSize == 0 {
		requestedPageSize = defaultXRefPageSize
	}

	eReply, err := g.Edges(ctx, &xpb.EdgesRequest{
		Ticket:    req.Ticket,
		PageSize:  int32(requestedPageSize),
		PageToken: req.PageToken,
	})
	if err != nil {
		return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
	}

	reply := &xpb.CrossReferencesReply{
		CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),

		NextPageToken: eReply.NextPageToken,
	}
	var allRelatedNodes stringset.Set
	if len(req.Filter) > 0 {
		reply.Nodes = make(map[string]*xpb.NodeInfo)
		allRelatedNodes = stringset.New()
	}

	// Cache parent files across all anchors
	files := make(map[string]*fileNode)

	var totalXRefs int
	for {
		for source, es := range eReply.EdgeSets {
			xr, ok := reply.CrossReferences[source]
			if !ok {
				xr = &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: source}
			}

			var count int
			for kind, grp := range es.Groups {
				switch {
				// TODO(schroeder): handle declarations
				case xrefs.IsDefKind(req.DefinitionKind, kind, false):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving definition anchors: %v", err)
					}
					count += len(anchors)
					xr.Definition = append(xr.Definition, anchors...)
				case xrefs.IsRefKind(req.ReferenceKind, kind):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving reference anchors: %v", err)
					}
					count += len(anchors)
					xr.Reference = append(xr.Reference, anchors...)
				case xrefs.IsDocKind(req.DocumentationKind, kind):
					anchors, err := completeAnchors(ctx, g, req.AnchorText, files, kind, edgeTickets(grp.Edge))
					if err != nil {
						return nil, fmt.Errorf("error resolving documentation anchors: %v", err)
					}
					count += len(anchors)
					xr.Documentation = append(xr.Documentation, anchors...)
				case allRelatedNodes != nil && !schema.IsAnchorEdge(kind):
					count += len(grp.Edge)
					for _, edge := range grp.Edge {
						xr.RelatedNode = append(xr.RelatedNode, &xpb.CrossReferencesReply_RelatedNode{
							Ticket:       edge.TargetTicket,
							RelationKind: kind,
							Ordinal:      edge.Ordinal,
						})
						allRelatedNodes.Add(edge.TargetTicket)
					}
				}
			}

			if count > 0 {
				reply.CrossReferences[xr.Ticket] = xr
				totalXRefs += count
			}
		}

		if reply.NextPageToken == "" || totalXRefs > 0 {
			break
		}

		// We need to return at least 1 xref, if there are any
		log.Println("Extra CrossReferences Edges call: ", reply.NextPageToken)
		eReply, err = g.Edges(ctx, &xpb.EdgesRequest{
			Ticket:    req.Ticket,
			PageSize:  int32(requestedPageSize),
			PageToken: reply.NextPageToken,
		})
		if err != nil {
			return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
		}
		reply.NextPageToken = eReply.NextPageToken
	}

	if len(allRelatedNodes) > 0 {
		nReply, err := g.Nodes(ctx, &xpb.NodesRequest{
			Ticket: allRelatedNodes.Slice(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("error retrieving related nodes: %v", err)
		}
		for ticket, n := range nReply.Nodes {
			reply.Nodes[ticket] = n
		}
	}

	return reply, nil
}
コード例 #3
0
ファイル: xrefs.go プロジェクト: baev/kythe
// CrossReferences returns the cross-references for the given tickets using the
// given NodesEdgesService.
func CrossReferences(ctx context.Context, xs NodesEdgesService, req *xpb.CrossReferencesRequest) (*xpb.CrossReferencesReply, error) {
	log.Println("WARNING: using experimental CrossReferences API")
	if len(req.Ticket) == 0 {
		return nil, errors.New("no cross-references requested")
	}

	requestedPageSize := int(req.PageSize)
	if requestedPageSize == 0 {
		requestedPageSize = defaultXRefPageSize
	}

	eReply, err := xs.Edges(ctx, &xpb.EdgesRequest{
		Ticket:    req.Ticket,
		PageSize:  int32(requestedPageSize),
		PageToken: req.PageToken,
	})
	if err != nil {
		return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
	}

	reply := &xpb.CrossReferencesReply{
		CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet),

		NextPageToken: eReply.NextPageToken,
	}
	var allRelatedNodes stringset.Set
	if len(req.Filter) > 0 {
		reply.Nodes = make(map[string]*xpb.NodeInfo)
		allRelatedNodes = stringset.New()
	}

	// Cache parent files across all anchors
	files := make(map[string]*fileNode)

	var totalXRefs int
	for {
		for _, es := range eReply.EdgeSet {
			xr, ok := reply.CrossReferences[es.SourceTicket]
			if !ok {
				xr = &xpb.CrossReferencesReply_CrossReferenceSet{Ticket: es.SourceTicket}
			}

			var count int
			for _, g := range es.Group {
				switch {
				case isDefKind(req.DefinitionKind, g.Kind):
					anchors, err := completeAnchors(ctx, xs, req.AnchorText, files, g.Kind, g.TargetTicket)
					if err != nil {
						return nil, fmt.Errorf("error resolving definition anchors: %v", err)
					}
					count += len(anchors)
					xr.Definition = append(xr.Definition, anchors...)
				case isRefKind(req.ReferenceKind, g.Kind):
					anchors, err := completeAnchors(ctx, xs, req.AnchorText, files, g.Kind, g.TargetTicket)
					if err != nil {
						return nil, fmt.Errorf("error resolving reference anchors: %v", err)
					}
					count += len(anchors)
					xr.Reference = append(xr.Reference, anchors...)
				case isDocKind(req.DocumentationKind, g.Kind):
					anchors, err := completeAnchors(ctx, xs, req.AnchorText, files, g.Kind, g.TargetTicket)
					if err != nil {
						return nil, fmt.Errorf("error resolving documentation anchors: %v", err)
					}
					count += len(anchors)
					xr.Documentation = append(xr.Documentation, anchors...)
				case allRelatedNodes != nil && !schema.IsAnchorEdge(g.Kind):
					count += len(g.TargetTicket)
					for _, target := range g.TargetTicket {
						xr.RelatedNode = append(xr.RelatedNode, &xpb.CrossReferencesReply_RelatedNode{
							Ticket:       target,
							RelationKind: g.Kind,
						})
					}
					allRelatedNodes.Add(g.TargetTicket...)
				}
			}

			if count > 0 {
				reply.CrossReferences[xr.Ticket] = xr
				totalXRefs += count
			}
		}

		if reply.NextPageToken == "" || totalXRefs > 0 {
			break
		}

		// We need to return at least 1 xref, if there are any
		log.Println("Extra CrossReferences Edges call: ", reply.NextPageToken)
		eReply, err = xs.Edges(ctx, &xpb.EdgesRequest{
			Ticket:    req.Ticket,
			PageSize:  int32(requestedPageSize),
			PageToken: reply.NextPageToken,
		})
		if err != nil {
			return nil, fmt.Errorf("error getting edges for cross-references: %v", err)
		}
		reply.NextPageToken = eReply.NextPageToken
	}

	if len(allRelatedNodes) > 0 {
		nReply, err := xs.Nodes(ctx, &xpb.NodesRequest{
			Ticket: allRelatedNodes.Slice(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("error retrieving related nodes: %v", err)
		}
		for _, n := range nReply.Node {
			reply.Nodes[n.Ticket] = n
		}
	}

	return reply, nil
}