Esempio n. 1
0
// SourceFromEntries returns a new Source from the given a set of entries with
// the same source VName.
func SourceFromEntries(entries []*spb.Entry) *Source {
	if len(entries) == 0 {
		return nil
	}

	src := &Source{
		Ticket: kytheuri.ToString(entries[0].Source),
		Facts:  make(map[string][]byte),
		Edges:  make(map[string][]string),
	}

	edgeTargets := make(map[string]stringset.Set)

	for _, e := range entries {
		if graphstore.IsEdge(e) {
			tgts, ok := edgeTargets[e.EdgeKind]
			if !ok {
				tgts = stringset.New()
				edgeTargets[e.EdgeKind] = tgts
			}
			tgts.Add(kytheuri.ToString(e.Target))
		} else {
			src.Facts[e.FactName] = e.FactValue
		}
	}
	for kind, targets := range edgeTargets {
		src.Edges[kind] = targets.Slice()
		sort.Strings(src.Edges[kind])
	}

	return src
}
Esempio n. 2
0
func TestDecorations(t *testing.T) {
	xs := newService(t, testEntries)

	reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
		Location: &xpb.Location{
			Ticket: kytheuri.ToString(testFileVName),
		},
		SourceText: true,
		References: true,
		Filter:     []string{"**"},
	})
	if err != nil {
		t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err)
	}

	if string(reply.SourceText) != testFileContent {
		t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent)
	}
	if reply.Encoding != testFileEncoding {
		t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding)
	}

	expectedRefs := []*xpb.DecorationsReply_Reference{
		{
			SourceTicket: kytheuri.ToString(testAnchorVName),
			TargetTicket: kytheuri.ToString(testAnchorTargetVName),
			Kind:         schema.RefEdge,
			AnchorStart: &xpb.Location_Point{
				ByteOffset:   1,
				LineNumber:   1,
				ColumnOffset: 1,
			},
			AnchorEnd: &xpb.Location_Point{
				ByteOffset:   4,
				LineNumber:   1,
				ColumnOffset: 4,
			},
		},
	}
	if err := testutil.DeepEqual(sortRefs(expectedRefs), sortRefs(reply.Reference)); err != nil {
		t.Error(err)
	}

	expectedNodes := nodesToInfos(testNodes[4:6])
	if err := testutil.DeepEqual(expectedNodes, sortInfos(reply.Node)); err != nil {
		t.Error(err)
	}
}
Esempio n. 3
0
func nodesToTickets(nodes []*node) []string {
	var tickets []string
	for _, n := range nodes {
		tickets = append(tickets, kytheuri.ToString(n.Source))
	}
	return tickets
}
Esempio n. 4
0
func nodesToInfos(nodes []*node) map[string]*cpb.NodeInfo {
	m := make(map[string]*cpb.NodeInfo)
	for _, n := range nodes {
		m[kytheuri.ToString(n.Source)] = n.Info()
	}
	return m
}
Esempio n. 5
0
func TestForPackage(t *testing.T) {
	tests := []struct {
		path   string
		ticket string
		isRoot bool
	}{
		{path: "bytes", ticket: "kythe://golang.org?lang=go?path=bytes#%3Apkg%3A", isRoot: true},
		{path: "go/types", ticket: "kythe://golang.org?lang=go?path=go/types#%3Apkg%3A", isRoot: true},
		{path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"},
		{path: "code.google.com/p/foo.bar/baz", ticket: "kythe://code.google.com/p/foo?lang=go?path=baz?root=bar#%3Apkg%3A"},
		{path: "fuzzy1.googlecode.com/alpha", ticket: "kythe://fuzzy1.googlecode.com?lang=go?path=alpha#%3Apkg%3A"},
		{path: "github.com/google/kythe/foo", ticket: "kythe://github.com/google/kythe?lang=go?path=foo#%3Apkg%3A"},
		{path: "bitbucket.org/zut/alors/non", ticket: "kythe://bitbucket.org/zut/alors?lang=go?path=non#%3Apkg%3A"},
		{path: "launchpad.net/~frood/blee/blor", ticket: "kythe://launchpad.net/~frood/blee?lang=go?path=blor#%3Apkg%3A"},

		{path: "golang.org/x/net/context", ticket: "kythe://golang.org/x/net?lang=go?path=context#%3Apkg%3A"},
	}
	for _, test := range tests {
		pkg := &build.Package{
			ImportPath: test.path,
			Goroot:     test.isRoot,
		}
		got := ForPackage("", pkg)
		gotTicket := kytheuri.ToString(got)
		if gotTicket != test.ticket {
			t.Errorf(`ForPackage("", [%s]): got %q, want %q`, test.path, gotTicket, test.ticket)
		}
	}
}
Esempio n. 6
0
func collectNodes(nodeEntries <-chan *spb.Entry) <-chan *srvpb.Node {
	nodes := make(chan *srvpb.Node)
	go func() {
		var (
			node  *srvpb.Node
			vname *spb.VName
		)
		for e := range nodeEntries {
			if node != nil && !compare.VNamesEqual(e.Source, vname) {
				nodes <- node
				node = nil
				vname = nil
			}
			if node == nil {
				vname = e.Source
				ticket := kytheuri.ToString(vname)
				node = &srvpb.Node{Ticket: ticket}
			}
			node.Fact = append(node.Fact, &srvpb.Node_Fact{
				Name:  e.FactName,
				Value: e.FactValue,
			})
		}
		if node != nil {
			nodes <- node
		}
		close(nodes)
	}()
	return nodes
}
Esempio n. 7
0
// Sources constructs a new Source for every contiguous set of entries sharing
// the same Source, calling f for each.
func Sources(rd stream.EntryReader, f func(*ipb.Source) error) error {
	var source *spb.VName
	var src *ipb.Source
	if err := rd(func(entry *spb.Entry) error {
		if src != nil && !compare.VNamesEqual(source, entry.Source) {
			if err := f(src); err != nil {
				return err
			}
			src = nil
		}
		if src == nil {
			source = entry.Source
			src = &ipb.Source{
				Ticket:     kytheuri.ToString(entry.Source),
				Facts:      make(map[string][]byte),
				EdgeGroups: make(map[string]*ipb.Source_EdgeGroup),
			}
		}
		AppendEntry(src, entry)
		return nil
	}); err != nil {
		return err
	}
	if src != nil {
		return f(src)
	}
	return nil
}
Esempio n. 8
0
// AppendEntry adds the given Entry to the Source's facts or edges.  It is
// assumed that src.Ticket == kytheuri.ToString(e.Source).
func AppendEntry(src *ipb.Source, e *spb.Entry) {
	if graphstore.IsEdge(e) {
		kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
		group, ok := src.EdgeGroups[kind]
		if !ok {
			group = &ipb.Source_EdgeGroup{}
			src.EdgeGroups[kind] = group
		}

		ticket := kytheuri.ToString(e.Target)

		ord := int32(ordinal)
		for _, edge := range group.Edges {
			if edge.Ticket == ticket && edge.Ordinal == ord {
				// Don't add duplicate edge
				return
			}
		}

		group.Edges = append(group.Edges, &ipb.Source_Edge{
			Ticket:  ticket,
			Ordinal: ord,
		})
	} else {
		src.Facts[e.FactName] = e.FactValue
	}
}
Esempio n. 9
0
func (n *node) EdgeSet() *xpb.EdgeSet {
	var groups []*xpb.EdgeSet_Group
	for kind, targets := range n.Edges {
		var tickets []string
		for _, target := range targets {
			tickets = append(tickets, kytheuri.ToString(target))
		}
		groups = append(groups, &xpb.EdgeSet_Group{
			Kind:         kind,
			TargetTicket: tickets,
		})
	}
	return &xpb.EdgeSet{
		SourceTicket: kytheuri.ToString(n.Source),
		Group:        groups,
	}
}
Esempio n. 10
0
func writeEdges(ctx context.Context, t table.Proto, edges <-chan *spb.Entry, maxEdgePageSize int) error {
	defer drainEntries(edges) // ensure channel is drained on errors

	temp, err := tempTable("edge.groups")
	if err != nil {
		return fmt.Errorf("failed to create temporary table: %v", err)
	}
	edgeGroups := &table.KVProto{temp}
	defer func() {
		if err := edgeGroups.Close(ctx); err != nil {
			log.Println("Error closing edge groups table: %v", err)
		}
	}()

	log.Println("Writing temporary edges table")

	var (
		src     *spb.VName
		kind    string
		targets stringset.Set
	)
	for e := range edges {
		if src != nil && (!compare.VNamesEqual(e.Source, src) || kind != e.EdgeKind) {
			if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
				return err
			}
			src = nil
		}
		if src == nil {
			src = e.Source
			kind = e.EdgeKind
			targets = stringset.New()
		}
		targets.Add(kytheuri.ToString(e.Target))
	}
	if src != nil {
		if err := writeWithReverses(ctx, edgeGroups, kytheuri.ToString(src), kind, targets.Slice()); err != nil {
			return err
		}
	}

	return writeEdgePages(ctx, t, edgeGroups, maxEdgePageSize)
}
Esempio n. 11
0
func nodesToEdgeSets(nodes []*node) map[string]*gpb.EdgeSet {
	sets := make(map[string]*gpb.EdgeSet)
	for _, n := range nodes {
		set := n.EdgeSet()
		if len(set.Groups) > 0 {
			sets[kytheuri.ToString(n.Source)] = set
		}
	}
	return sets
}
Esempio n. 12
0
func (n *node) Info() *xpb.NodeInfo {
	info := &xpb.NodeInfo{
		Ticket: kytheuri.ToString(n.Source),
	}
	for name, val := range n.Facts {
		info.Fact = append(info.Fact, &xpb.Fact{
			Name:  name,
			Value: []byte(val),
		})
	}
	return info
}
Esempio n. 13
0
func node(v *spb.VName, facts ...string) *srvpb.Node {
	if len(facts)%2 != 0 {
		panic("odd number of facts")
	}
	n := &srvpb.Node{Ticket: kytheuri.ToString(v)}
	for i := 0; i < len(facts); i += 2 {
		n.Fact = append(n.Fact, &srvpb.Node_Fact{
			Name:  facts[i],
			Value: []byte(facts[i+1]),
		})
	}
	return n
}
Esempio n. 14
0
func TestDecorations(t *testing.T) {
	xs := newService(t, testEntries)

	reply, err := xs.Decorations(ctx, &xpb.DecorationsRequest{
		Location: &xpb.Location{
			Ticket: kytheuri.ToString(testFileVName),
		},
		SourceText: true,
		References: true,
	})
	if err != nil {
		t.Fatalf("Error fetching decorations for %+v: %v", testFileVName, err)
	}

	if string(reply.SourceText) != testFileContent {
		t.Errorf("Incorrect file content: %q; Expected: %q", string(reply.SourceText), testFileContent)
	}
	if reply.Encoding != testFileEncoding {
		t.Errorf("Incorrect file encoding: %q; Expected: %q", reply.Encoding, testFileEncoding)
	}

	expectedRefs := []*xpb.DecorationsReply_Reference{
		{
			SourceTicket: kytheuri.ToString(testAnchorVName),
			TargetTicket: kytheuri.ToString(testAnchorTargetVName),
			Kind:         schema.RefEdge,
		},
	}
	if !reflect.DeepEqual(sortRefs(reply.Reference), sortRefs(expectedRefs)) {
		t.Errorf("Got %v; Expected references %v", reply.Reference, expectedRefs)
	}

	refNodes := testNodes[4:6]
	expectedNodes := nodesToInfos(refNodes)
	if !reflect.DeepEqual(sortInfos(reply.Node), expectedNodes) {
		t.Errorf("Got %v; Expected nodes %v", reply.Node, expectedNodes)
	}
}
Esempio n. 15
0
func (n *node) EdgeSet() *gpb.EdgeSet {
	groups := make(map[string]*gpb.EdgeSet_Group)
	for kind, targets := range n.Edges {
		var edges []*gpb.EdgeSet_Group_Edge
		for ordinal, target := range targets {
			edges = append(edges, &gpb.EdgeSet_Group_Edge{
				TargetTicket: kytheuri.ToString(target),
				Ordinal:      int32(ordinal),
			})
		}
		groups[kind] = &gpb.EdgeSet_Group{
			Edge: edges,
		}
	}
	return &gpb.EdgeSet{
		Groups: groups,
	}
}
Esempio n. 16
0
// SourceFromEntries returns a new Source from the given a set of entries with
// the same source VName.
func SourceFromEntries(entries []*spb.Entry) *ipb.Source {
	if len(entries) == 0 {
		return nil
	}

	src := &ipb.Source{
		Ticket:     kytheuri.ToString(entries[0].Source),
		Facts:      make(map[string][]byte),
		EdgeGroups: make(map[string]*ipb.Source_EdgeGroup),
	}

	for _, e := range entries {
		AppendEntry(src, e)
	}

	for _, group := range src.EdgeGroups {
		sort.Sort(byOrdinal(group.Edges))
	}

	return src
}
Esempio n. 17
0
// AddFile adds the given file VName to m.
func (m *Map) AddFile(file *spb.VName) {
	ticket := kytheuri.ToString(file)
	dirPath := CleanDirPath(path.Dir(file.Path))
	dir := m.ensureDir(file.Corpus, file.Root, dirPath)
	dir.File = addToSet(dir.File, ticket)
}
Esempio n. 18
0
// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB) error {
	log.Println("Starting serving pipeline")
	tbl := &table.KVProto{db}

	// TODO(schroederc): for large corpora, this won't fit in memory
	var files []string

	entries := make(chan *spb.Entry)
	ftIn, nIn, eIn := make(chan *spb.VName), make(chan *spb.Entry), make(chan *spb.Entry)
	go func() {
		for entry := range entries {
			if entry.EdgeKind == "" {
				nIn <- entry
				if entry.FactName == schema.NodeKindFact && string(entry.FactValue) == "file" {
					ftIn <- entry.Source
					files = append(files, kytheuri.ToString(entry.Source))
				}
			} else {
				eIn <- entry
			}
		}
		close(ftIn)
		close(nIn)
		close(eIn)
	}()
	log.Println("Scanning GraphStore")
	var sErr error
	go func() {
		sErr = gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
			entries <- e
			return nil
		})
		close(entries)
	}()

	var (
		ftErr, nErr, eErr error
		ftWG, edgeNodeWG  sync.WaitGroup
	)
	ftWG.Add(1)
	go func() {
		defer ftWG.Done()
		ftErr = writeFileTree(ctx, tbl, ftIn)
		log.Println("Wrote FileTree")
	}()
	edgeNodeWG.Add(2)
	nodes := make(chan *srvpb.Node)
	go func() {
		defer edgeNodeWG.Done()
		nErr = writeNodes(tbl, nIn, nodes)
		log.Println("Wrote Nodes")
	}()
	go func() {
		defer edgeNodeWG.Done()
		eErr = writeEdges(ctx, tbl, eIn)
		log.Println("Wrote Edges")
	}()

	var (
		idxWG  sync.WaitGroup
		idxErr error
	)
	idxWG.Add(1)
	go func() {
		defer idxWG.Done()
		idxErr = writeIndex(&table.KVInverted{db}, nodes)
		log.Println("Wrote Search Index")
	}()

	edgeNodeWG.Wait()
	if eErr != nil {
		return eErr
	} else if nErr != nil {
		return nErr
	}

	es := xrefs.NodesEdgesService(&xsrv.Table{tbl})
	if err := writeDecorations(ctx, tbl, es, files); err != nil {
		return err
	}

	ftWG.Wait()
	if ftErr != nil {
		return ftErr
	}
	idxWG.Wait()
	if idxErr != nil {
		return idxErr
	}

	return sErr
}
Esempio n. 19
0
func writeEdgePages(ctx context.Context, t table.Proto, gs graphstore.Service) error {
	// TODO(schroederc): spill large PagedEdgeSets into EdgePages
	log.Println("Writing EdgeSets")
	var (
		lastSrc  *spb.VName
		pes      *srvpb.PagedEdgeSet
		grp      *srvpb.EdgeSet_Group
		pesTotal int
	)
	if err := gs.Scan(ctx, new(spb.ScanRequest), func(e *spb.Entry) error {
		if e.EdgeKind == "" {
			panic("non-edge entry")
		}

		if pes != nil && !compare.VNamesEqual(lastSrc, e.Source) {
			if grp != nil {
				pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
				pesTotal += len(grp.TargetTicket)
			}
			pes.TotalEdges = int32(pesTotal)
			if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
				return err
			}
			pes = nil
			grp = nil
			pesTotal = 0
		}
		if pes == nil {
			pes = &srvpb.PagedEdgeSet{
				EdgeSet: &srvpb.EdgeSet{
					SourceTicket: kytheuri.ToString(e.Source),
				},
			}
		}

		if grp != nil && grp.Kind != e.EdgeKind {
			pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
			pesTotal += len(grp.TargetTicket)
			grp = nil
		}
		if grp == nil {
			grp = &srvpb.EdgeSet_Group{
				Kind: e.EdgeKind,
			}
		}

		grp.TargetTicket = append(grp.TargetTicket, kytheuri.ToString(e.Target))
		lastSrc = e.Source
		return nil
	}); err != nil {
		return err
	}
	if pes != nil {
		if grp != nil {
			pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
			pesTotal += len(grp.TargetTicket)
		}
		pes.TotalEdges = int32(pesTotal)
		if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 20
0
// Edges implements part of the Service interface.
func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) {
	if len(req.Ticket) == 0 {
		return nil, errors.New("no tickets specified")
	} else if req.PageToken != "" {
		return nil, errors.New("UNIMPLEMENTED: page_token")
	}

	patterns := xrefs.ConvertFilters(req.Filter)
	allowedKinds := stringset.New(req.Kind...)
	targetSet := stringset.New()
	reply := new(xpb.EdgesReply)

	for _, ticket := range req.Ticket {
		vname, err := kytheuri.ToVName(ticket)
		if err != nil {
			return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err)
		}

		var (
			// EdgeKind -> StringSet<TargetTicket>
			filteredEdges = make(map[string]stringset.Set)
			filteredFacts []*xpb.Fact
		)

		if err := g.gs.Read(ctx, &spb.ReadRequest{
			Source:   vname,
			EdgeKind: "*",
		}, func(entry *spb.Entry) error {
			edgeKind := entry.EdgeKind
			if edgeKind == "" {
				// node fact
				if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) {
					filteredFacts = append(filteredFacts, entryToFact(entry))
				}
			} else {
				// edge
				if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) {
					targets := filteredEdges[edgeKind]
					if targets == nil {
						targets = stringset.New()
						filteredEdges[edgeKind] = targets
					}
					targets.Add(kytheuri.ToString(entry.Target))
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket)
		}

		// Only add a EdgeSet if there are targets for the requested edge kinds.
		if len(filteredEdges) > 0 {
			var groups []*xpb.EdgeSet_Group
			for edgeKind, targets := range filteredEdges {
				g := &xpb.EdgeSet_Group{Kind: edgeKind}
				for target := range targets {
					g.TargetTicket = append(g.TargetTicket, target)
					targetSet.Add(target)
				}
				groups = append(groups, g)
			}
			reply.EdgeSet = append(reply.EdgeSet, &xpb.EdgeSet{
				SourceTicket: ticket,
				Group:        groups,
			})

			// In addition, only add a NodeInfo if the filters have resulting facts.
			if len(filteredFacts) > 0 {
				reply.Node = append(reply.Node, &xpb.NodeInfo{
					Ticket: ticket,
					Fact:   filteredFacts,
				})
			}
		}
	}

	// Only request Nodes when there are fact filters given.
	if len(req.Filter) > 0 {
		// Ensure reply.Node is a unique set by removing already requested nodes from targetSet
		for _, n := range reply.Node {
			targetSet.Remove(n.Ticket)
		}

		// Batch request all leftover target nodes
		nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
			Ticket: targetSet.Slice(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("failure getting target nodes: %v", err)
		}
		reply.Node = append(reply.Node, nodesReply.Node...)
	}

	return reply, nil
}
Esempio n. 21
0
func (d *DB) copyEntries(entries <-chan *spb.Entry) error {
	// Start a transaction for a COPY statement per table
	nodesTx, err := d.Begin()
	if err != nil {
		return err
	}
	edgesTx, err := d.Begin()
	if err != nil {
		return err
	}

	// Create each table in their corresponding transactions to speed up COPY
	if _, err := nodesTx.Exec(createNodesTable); err != nil {
		return fmt.Errorf("error truncating Nodes table: %v", err)
	} else if _, err := edgesTx.Exec(createEdgeTable); err != nil {
		return fmt.Errorf("error truncating Edges table: %v", err)
	}

	copyNode, err := nodesTx.Prepare(pq.CopyIn(
		"nodes",
		"ticket",
		"node_kind",
		"subkind",
		"text",
		"text_encoding",
		"start_offset",
		"end_offset",
		"snippet_start",
		"snippet_end",
		"other_facts_num",
		"other_facts",
	))
	if err != nil {
		return fmt.Errorf("error preparing Nodes copy: %v", err)
	}

	copyEdge, err := edgesTx.Prepare(pq.CopyIn(
		"edges",
		"source",
		"kind",
		"target",
		"ordinal",
	))
	if err != nil {
		return fmt.Errorf("error preparing Edges copy: %v", err)
	}

	var node srvpb.Node
	var nodeKind string
	var subkind, textEncoding *string
	var text *[]byte
	var startOffset, endOffset, snippetStart, snippetEnd *int64
	for e := range entries {
		if graphstore.IsNodeFact(e) {
			ticket := kytheuri.ToString(e.Source)
			if node.Ticket != "" && node.Ticket != ticket {
				nodeTicket := node.Ticket
				node.Ticket = ""
				var rec []byte
				if len(node.Fact) > 0 {
					rec, err = proto.Marshal(&node)
					if err != nil {
						return fmt.Errorf("error marshaling facts: %v", err)
					}
				}
				if text != nil && textEncoding == nil {
					textEncoding = proto.String(facts.DefaultTextEncoding)
				}
				if _, err := copyNode.Exec(
					nodeTicket,
					nodeKind,
					subkind,
					text,
					textEncoding,
					startOffset,
					endOffset,
					snippetStart,
					snippetEnd,
					len(node.Fact),
					rec,
				); err != nil {
					return fmt.Errorf("error copying node: %v", err)
				}
				node.Fact, text = node.Fact[0:0], nil
				nodeKind = ""
				subkind, textEncoding = nil, nil
				startOffset, endOffset, snippetStart, snippetEnd = nil, nil, nil, nil
			}
			if node.Ticket == "" {
				node.Ticket = ticket
			}
			switch e.FactName {
			case facts.NodeKind:
				nodeKind = string(e.FactValue)
			case facts.Subkind:
				subkind = proto.String(string(e.FactValue))
			case facts.Text:
				text = &e.FactValue
			case facts.TextEncoding:
				textEncoding = proto.String(string(e.FactValue))
			case facts.AnchorStart:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					startOffset = proto.Int64(n)
				}
			case facts.AnchorEnd:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					endOffset = proto.Int64(n)
				}
			case facts.SnippetStart:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					snippetStart = proto.Int64(n)
				}
			case facts.SnippetEnd:
				n, err := strconv.ParseInt(string(e.FactValue), 10, 64)
				if err == nil {
					snippetEnd = proto.Int64(n)
				}
			default:
				node.Fact = append(node.Fact, &cpb.Fact{
					Name:  e.FactName,
					Value: e.FactValue,
				})
			}
		} else if edges.IsForward(e.EdgeKind) {
			kind, ordinal, _ := edges.ParseOrdinal(e.EdgeKind)
			ticket := kytheuri.ToString(e.Source)
			if _, err := copyEdge.Exec(ticket, kind, kytheuri.ToString(e.Target), ordinal); err != nil {
				return fmt.Errorf("error copying edge: %v", err)
			}
		}
	}

	if _, err := copyNode.Exec(); err != nil {
		return fmt.Errorf("error flushing nodes: %v", err)
	} else if _, err := copyEdge.Exec(); err != nil {
		return fmt.Errorf("error flushing edges: %v", err)
	}

	if err := nodesTx.Commit(); err != nil {
		return fmt.Errorf("error committing Nodes transaction: %v", err)
	} else if err := edgesTx.Commit(); err != nil {
		return fmt.Errorf("error committing Edges transaction: %v", err)
	}

	return nil
}
Esempio n. 22
0
File: xrefs.go Progetto: bzz/kythe
// Edges implements part of the Service interface.
func (g *GraphStoreService) Edges(ctx context.Context, req *xpb.EdgesRequest) (*xpb.EdgesReply, error) {
	if len(req.Ticket) == 0 {
		return nil, errors.New("no tickets specified")
	} else if req.PageToken != "" {
		return nil, errors.New("UNIMPLEMENTED: page_token")
	}

	patterns := xrefs.ConvertFilters(req.Filter)
	allowedKinds := stringset.New(req.Kind...)
	targetSet := stringset.New()
	reply := &xpb.EdgesReply{
		EdgeSets: make(map[string]*xpb.EdgeSet),
		Nodes:    make(map[string]*xpb.NodeInfo),
	}

	for _, ticket := range req.Ticket {
		vname, err := kytheuri.ToVName(ticket)
		if err != nil {
			return nil, fmt.Errorf("invalid ticket %q: %v", ticket, err)
		}

		var (
			// EdgeKind -> TargetTicket -> OrdinalSet
			filteredEdges = make(map[string]map[string]map[int32]struct{})
			filteredFacts = make(map[string][]byte)
		)

		if err := g.gs.Read(ctx, &spb.ReadRequest{
			Source:   vname,
			EdgeKind: "*",
		}, func(entry *spb.Entry) error {
			edgeKind := entry.EdgeKind
			if edgeKind == "" {
				// node fact
				if len(patterns) > 0 && xrefs.MatchesAny(entry.FactName, patterns) {
					filteredFacts[entry.FactName] = entry.FactValue
				}
			} else {
				// edge
				edgeKind, ordinal, _ := schema.ParseOrdinal(edgeKind)
				if len(req.Kind) == 0 || allowedKinds.Contains(edgeKind) {
					targets, ok := filteredEdges[edgeKind]
					if !ok {
						targets = make(map[string]map[int32]struct{})
						filteredEdges[edgeKind] = targets
					}
					ticket := kytheuri.ToString(entry.Target)
					ordSet, ok := targets[ticket]
					if !ok {
						ordSet = make(map[int32]struct{})
						targets[ticket] = ordSet
					}
					ordSet[int32(ordinal)] = struct{}{}
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("failed to retrieve entries for ticket %q", ticket)
		}

		// Only add a EdgeSet if there are targets for the requested edge kinds.
		if len(filteredEdges) > 0 {
			groups := make(map[string]*xpb.EdgeSet_Group)
			for edgeKind, targets := range filteredEdges {
				g := &xpb.EdgeSet_Group{}
				for target, ordinals := range targets {
					for ordinal := range ordinals {
						g.Edge = append(g.Edge, &xpb.EdgeSet_Group_Edge{
							TargetTicket: target,
							Ordinal:      ordinal,
						})
					}
					targetSet.Add(target)
				}
				groups[edgeKind] = g
			}
			reply.EdgeSets[ticket] = &xpb.EdgeSet{
				Groups: groups,
			}

			// In addition, only add a NodeInfo if the filters have resulting facts.
			if len(filteredFacts) > 0 {
				reply.Nodes[ticket] = &xpb.NodeInfo{
					Facts: filteredFacts,
				}
			}
		}
	}

	// Only request Nodes when there are fact filters given.
	if len(req.Filter) > 0 {
		// Eliminate redundant work by removing already requested nodes from targetSet
		for ticket := range reply.Nodes {
			targetSet.Remove(ticket)
		}

		// Batch request all leftover target nodes
		nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
			Ticket: targetSet.Slice(),
			Filter: req.Filter,
		})
		if err != nil {
			return nil, fmt.Errorf("failure getting target nodes: %v", err)
		}
		for ticket, node := range nodesReply.Nodes {
			reply.Nodes[ticket] = node
		}
	}

	return reply, nil
}
Esempio n. 23
0
File: xrefs.go Progetto: bzz/kythe
// Decorations implements part of the Service interface.
func (g *GraphStoreService) Decorations(ctx context.Context, req *xpb.DecorationsRequest) (*xpb.DecorationsReply, error) {
	if len(req.DirtyBuffer) > 0 {
		return nil, errors.New("UNIMPLEMENTED: dirty buffers")
	} else if req.GetLocation() == nil {
		// TODO(schroederc): allow empty location when given dirty buffer
		return nil, errors.New("missing location")
	}

	fileVName, err := kytheuri.ToVName(req.Location.Ticket)
	if err != nil {
		return nil, fmt.Errorf("invalid file ticket %q: %v", req.Location.Ticket, err)
	}

	text, encoding, err := getSourceText(ctx, g.gs, fileVName)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve file text: %v", err)
	}
	norm := xrefs.NewNormalizer(text)

	loc, err := norm.Location(req.GetLocation())
	if err != nil {
		return nil, err
	}

	reply := &xpb.DecorationsReply{
		Location: loc,
		Nodes:    make(map[string]*xpb.NodeInfo),
	}

	// Handle DecorationsRequest.SourceText switch
	if req.SourceText {
		if loc.Kind == xpb.Location_FILE {
			reply.SourceText = text
		} else {
			reply.SourceText = text[loc.Start.ByteOffset:loc.End.ByteOffset]
		}
		reply.Encoding = encoding
	}

	// Handle DecorationsRequest.References switch
	if req.References {
		// Traverse the following chain of edges:
		//   file --%/kythe/edge/childof-> []anchor --forwardEdgeKind-> []target
		//
		// Add []anchor and []target nodes to reply.Nodes
		// Add all {anchor, forwardEdgeKind, target} tuples to reply.Reference

		patterns := xrefs.ConvertFilters(req.Filter)

		children, err := getEdges(ctx, g.gs, fileVName, func(e *spb.Entry) bool {
			return e.EdgeKind == revChildOfEdgeKind
		})
		if err != nil {
			return nil, fmt.Errorf("failed to retrieve file children: %v", err)
		}

		targetSet := stringset.New()
		for _, edge := range children {
			anchor := edge.Target
			ticket := kytheuri.ToString(anchor)
			anchorNodeReply, err := g.Nodes(ctx, &xpb.NodesRequest{
				Ticket: []string{ticket},
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference source node: %v", err)
			} else if len(anchorNodeReply.Nodes) != 1 {
				return nil, fmt.Errorf("found %d nodes for {%+v}", len(anchorNodeReply.Nodes), anchor)
			}

			node, ok := xrefs.NodesMap(anchorNodeReply.Nodes)[ticket]
			if !ok {
				return nil, fmt.Errorf("failed to find info for node %q", ticket)
			} else if string(node[schema.NodeKindFact]) != schema.AnchorKind {
				// Skip child if it isn't an anchor node
				continue
			}

			anchorStart, err := strconv.Atoi(string(node[schema.AnchorStartFact]))
			if err != nil {
				log.Printf("Invalid anchor start offset %q for node %q: %v", node[schema.AnchorStartFact], ticket, err)
				continue
			}
			anchorEnd, err := strconv.Atoi(string(node[schema.AnchorEndFact]))
			if err != nil {
				log.Printf("Invalid anchor end offset %q for node %q: %v", node[schema.AnchorEndFact], ticket, err)
				continue
			}

			if loc.Kind == xpb.Location_SPAN {
				// Check if anchor fits within/around requested source text window
				if !xrefs.InSpanBounds(req.SpanKind, int32(anchorStart), int32(anchorEnd), loc.Start.ByteOffset, loc.End.ByteOffset) {
					continue
				} else if anchorStart > anchorEnd {
					log.Printf("Invalid anchor offset span %d:%d", anchorStart, anchorEnd)
					continue
				}
			}

			targets, err := getEdges(ctx, g.gs, anchor, func(e *spb.Entry) bool {
				return schema.EdgeDirection(e.EdgeKind) == schema.Forward && e.EdgeKind != schema.ChildOfEdge
			})
			if err != nil {
				return nil, fmt.Errorf("failed to retrieve targets of anchor %v: %v", anchor, err)
			}
			if len(targets) == 0 {
				log.Printf("Anchor missing forward edges: {%+v}", anchor)
				continue
			}

			if node := filterNode(patterns, anchorNodeReply.Nodes[ticket]); node != nil {
				reply.Nodes[ticket] = node
			}
			for _, edge := range targets {
				targetTicket := kytheuri.ToString(edge.Target)
				targetSet.Add(targetTicket)
				reply.Reference = append(reply.Reference, &xpb.DecorationsReply_Reference{
					SourceTicket: ticket,
					Kind:         edge.Kind,
					TargetTicket: targetTicket,
					AnchorStart:  norm.ByteOffset(int32(anchorStart)),
					AnchorEnd:    norm.ByteOffset(int32(anchorEnd)),
				})
			}
		}
		sort.Sort(bySpan(reply.Reference))

		// Only request Nodes when there are fact filters given.
		if len(req.Filter) > 0 {
			// Ensure returned nodes are not duplicated.
			for ticket := range reply.Nodes {
				targetSet.Remove(ticket)
			}

			// Batch request all Reference target nodes
			nodesReply, err := g.Nodes(ctx, &xpb.NodesRequest{
				Ticket: targetSet.Slice(),
				Filter: req.Filter,
			})
			if err != nil {
				return nil, fmt.Errorf("failure getting reference target nodes: %v", err)
			}
			for ticket, node := range nodesReply.Nodes {
				reply.Nodes[ticket] = node
			}
		}
	}

	return reply, nil
}
Esempio n. 24
0
// AddFile adds the given file VName to m.
func (m *Map) AddFile(file *spb.VName) {
	ticket := kytheuri.ToString(file)
	path := filepath.Join("/", file.Path)
	dir := m.ensureDir(file.Corpus, file.Root, filepath.Dir(path))
	dir.File = addToSet(dir.File, ticket)
}