func writeNodes(t table.Proto, nodeEntries <-chan *spb.Entry, nodes chan<- *srvpb.Node) error {
defer close(nodes)
for node := range collectNodes(nodeEntries) {
nodes <- node
if err := t.Put(xsrv.NodeKey(node.Ticket), node); err != nil {
return err
}
}
return nil
}
func writeNodes(ctx context.Context, t table.Proto, nodeEntries <-chan *spb.Entry, nodes chan<- *srvpb.Node) error {
defer close(nodes)
defer drainEntries(nodeEntries) // ensure channel is drained on errors
for node := range collectNodes(nodeEntries) {
nodes <- node
if err := t.Put(ctx, xrefs.NodeKey(node.Ticket), node); err != nil {
return err
}
}
return nil
}
func writeDecorations(ctx context.Context, t table.Proto, es xrefs.NodesEdgesService, files []string) error {
log.Println("Writing Decorations")
edges := make(chan *xpb.EdgesReply)
var eErr error
go func() {
eErr = readEdges(ctx, es, files, edges,
decorationFilters, []string{revChildOfEdgeKind})
close(edges)
}()
for e := range edges {
decor := &srvpb.FileDecorations{}
if len(e.EdgeSet) == 0 {
if len(e.Node) != 1 {
log.Println("ERROR: missing node for non-decoration file")
continue
}
decor.FileTicket = e.Node[0].Ticket
} else if len(e.EdgeSet) != 1 {
log.Println("ERROR: invalid number of decoration EdgeSets:", len(e.EdgeSet))
continue
} else {
decor.FileTicket = e.EdgeSet[0].SourceTicket
}
for _, n := range e.Node {
if n.Ticket == decor.FileTicket {
for _, f := range n.Fact {
switch f.Name {
case schema.TextFact:
decor.SourceText = f.Value
case schema.TextEncodingFact:
decor.Encoding = string(f.Value)
}
}
} else {
ds, err := getDecorations(ctx, es, n)
if err != nil {
return err
}
decor.Decoration = append(decor.Decoration, ds...)
}
}
sort.Sort(byOffset(decor.Decoration))
if err := t.Put(xsrv.DecorationsKey(decor.FileTicket), decor); err != nil {
return err
}
}
return eErr
}
func writePagedEdges(ctx context.Context, edges <-chan *srvpb.Edge, out table.Proto, opts *Options) error {
buffer := out.Buffered()
log.Println("Writing EdgeSets")
esb := &assemble.EdgeSetBuilder{
MaxEdgePageSize: opts.MaxPageSize,
Output: func(ctx context.Context, pes *srvpb.PagedEdgeSet) error {
return buffer.Put(ctx, xsrv.EdgeSetKey(pes.Source.Ticket), pes)
},
OutputPage: func(ctx context.Context, ep *srvpb.EdgePage) error {
return buffer.Put(ctx, xsrv.EdgePageKey(ep.PageKey), ep)
},
}
var grp *srvpb.EdgeGroup
for e := range edges {
if grp != nil && (e.Target == nil || grp.Kind != e.Kind) {
if err := esb.AddGroup(ctx, grp); err != nil {
for range edges {
} // drain input channel
return err
}
grp = nil
}
if e.Target == nil {
// Head-only edge: signals a new set of edges with the same Source
if err := esb.StartEdgeSet(ctx, e.Source); err != nil {
return err
}
} else if grp == nil {
grp = &srvpb.EdgeGroup{
Kind: e.Kind,
Target: []*srvpb.Node{e.Target},
}
} else {
grp.Target = append(grp.Target, e.Target)
}
}
if grp != nil {
if err := esb.AddGroup(ctx, grp); err != nil {
return err
}
}
if err := esb.Flush(ctx); err != nil {
return err
}
return buffer.Flush(ctx)
}
func writeFileTree(ctx context.Context, tree *filetree.Map, out table.Proto) error {
for corpus, roots := range tree.M {
for root, dirs := range roots {
for path, dir := range dirs {
if err := out.Put(ctx, ftsrv.DirKey(corpus, root, path), dir); err != nil {
return err
}
}
}
}
cr, err := tree.CorpusRoots(ctx, &ftpb.CorpusRootsRequest{})
if err != nil {
return err
}
return out.Put(ctx, ftsrv.CorpusRootsKey, cr)
}
func writeEdgePages(ctx context.Context, t table.Proto, edgeGroups *table.KVProto, maxEdgePageSize int) error {
// TODO(schroederc): spill large PagedEdgeSets into EdgePages
it, err := edgeGroups.ScanPrefix(nil, &keyvalue.Options{LargeRead: true})
if err != nil {
return err
}
defer it.Close()
log.Println("Writing EdgeSets")
esb := &assemble.EdgeSetBuilder{
MaxEdgePageSize: maxEdgePageSize,
Output: func(ctx context.Context, pes *srvpb.PagedEdgeSet) error {
return t.Put(ctx, xrefs.EdgeSetKey(pes.EdgeSet.SourceTicket), pes)
},
OutputPage: func(ctx context.Context, ep *srvpb.EdgePage) error {
return t.Put(ctx, xrefs.EdgePageKey(ep.PageKey), ep)
},
}
for {
k, v, err := it.Next()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("error scanning edge groups table: %v", err)
}
ss := strings.Split(string(k), tempTableKeySep)
if len(ss) != 3 {
return fmt.Errorf("invalid edge groups table key: %q", string(k))
}
src := ss[0]
var eg srvpb.EdgeSet_Group
if err := proto.Unmarshal(v, &eg); err != nil {
return fmt.Errorf("invalid edge groups table value: %v", err)
}
if err := esb.AddGroup(ctx, src, &eg); err != nil {
return err
}
}
return esb.Flush(ctx)
}
func writeFileTree(ctx context.Context, tree *filetree.Map, out table.Proto) error {
buffer := out.Buffered()
for corpus, roots := range tree.M {
for root, dirs := range roots {
for path, dir := range dirs {
if err := buffer.Put(ctx, ftsrv.PrefixedDirKey(corpus, root, path), dir); err != nil {
return err
}
}
}
}
cr, err := tree.CorpusRoots(ctx, &ftpb.CorpusRootsRequest{})
if err != nil {
return err
}
if err := buffer.Put(ctx, ftsrv.CorpusRootsPrefixedKey, cr); err != nil {
return err
}
return buffer.Flush(ctx)
}
func writeFileTree(ctx context.Context, t table.Proto, files <-chan *spb.VName) error {
tree := filetree.NewMap()
for f := range files {
tree.AddFile(f)
// TODO(schroederc): evict finished directories (based on GraphStore order)
}
for corpus, roots := range tree.M {
for root, dirs := range roots {
for path, dir := range dirs {
if err := t.Put(ftsrv.DirKey(corpus, root, path), dir); err != nil {
return err
}
}
}
}
cr, err := tree.CorpusRoots(ctx, &ftpb.CorpusRootsRequest{})
if err != nil {
return err
}
return t.Put(ftsrv.CorpusRootsKey, cr)
}
func writeEdgePages(ctx context.Context, t table.Proto, gs graphstore.Service) error {
// TODO(schroederc): spill large PagedEdgeSets into EdgePages
log.Println("Writing EdgeSets")
var (
lastSrc *spb.VName
pes *srvpb.PagedEdgeSet
grp *srvpb.EdgeSet_Group
pesTotal int
)
if err := gs.Scan(ctx, new(spb.ScanRequest), func(e *spb.Entry) error {
if e.EdgeKind == "" {
panic("non-edge entry")
}
if pes != nil && !compare.VNamesEqual(lastSrc, e.Source) {
if grp != nil {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
}
pes.TotalEdges = int32(pesTotal)
if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
return err
}
pes = nil
grp = nil
pesTotal = 0
}
if pes == nil {
pes = &srvpb.PagedEdgeSet{
EdgeSet: &srvpb.EdgeSet{
SourceTicket: kytheuri.ToString(e.Source),
},
}
}
if grp != nil && grp.Kind != e.EdgeKind {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
grp = nil
}
if grp == nil {
grp = &srvpb.EdgeSet_Group{
Kind: e.EdgeKind,
}
}
grp.TargetTicket = append(grp.TargetTicket, kytheuri.ToString(e.Target))
lastSrc = e.Source
return nil
}); err != nil {
return err
}
if pes != nil {
if grp != nil {
pes.EdgeSet.Group = append(pes.EdgeSet.Group, grp)
pesTotal += len(grp.TargetTicket)
}
pes.TotalEdges = int32(pesTotal)
if err := t.Put(xsrv.EdgeSetKey(pes.EdgeSet.SourceTicket), pes); err != nil {
return err
}
}
return nil
}
func writeDecor(ctx context.Context, t table.Proto, decor *srvpb.FileDecorations) error {
sort.Sort(assemble.ByOffset(decor.Decoration))
return t.Put(ctx, xrefs.DecorationsKey(decor.FileTicket), decor)
}
