func combineNodesAndEdges(ctx context.Context, opts *Options, out *servingOutput, gsEntries <-chan *spb.Entry) (disksort.Interface, error) {
log.Println("Writing partial edges")
tree := filetree.NewMap()
partialSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
if err != nil {
return nil, err
}
bIdx := out.idx.Buffered()
var src *spb.VName
var entries []*spb.Entry
for e := range gsEntries {
if e.FactName == schema.NodeKindFact && string(e.FactValue) == schema.FileKind {
tree.AddFile(e.Source)
// TODO(schroederc): evict finished directories (based on GraphStore order)
}
if src == nil {
src = e.Source
} else if !compare.VNamesEqual(e.Source, src) {
if err := writePartialEdges(ctx, partialSorter, bIdx, assemble.SourceFromEntries(entries)); err != nil {
drainEntries(gsEntries)
return nil, err
}
src = e.Source
entries = nil
}
entries = append(entries, e)
}
if len(entries) > 0 {
if err := writePartialEdges(ctx, partialSorter, bIdx, assemble.SourceFromEntries(entries)); err != nil {
return nil, err
}
}
if err := bIdx.Flush(ctx); err != nil {
return nil, err
}
if err := writeFileTree(ctx, tree, out.xs); err != nil {
return nil, fmt.Errorf("error writing file tree: %v", err)
}
tree = nil
log.Println("Writing complete edges")
cSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
if err != nil {
return nil, err
}
var n *srvpb.Node
if err := partialSorter.Read(func(i interface{}) error {
e := i.(*srvpb.Edge)
if n == nil || n.Ticket != e.Source.Ticket {
n = e.Source
return cSorter.Add(e)
} else if e.Target != nil {
e.Source = n
if err := writeCompletedEdges(ctx, cSorter, e); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("error reading/writing edges: %v", err)
}
return cSorter, nil
}
func combineNodesAndEdges(ctx context.Context, out *servingOutput, gsEntries <-chan *spb.Entry) error {
log.Println("Writing partial edges")
tree := filetree.NewMap()
var src *spb.VName
var entries []*spb.Entry
for e := range gsEntries {
if e.FactName == schema.NodeKindFact && string(e.FactValue) == schema.FileKind {
tree.AddFile(e.Source)
// TODO(schroederc): evict finished directories (based on GraphStore order)
}
if src == nil {
src = e.Source
} else if !compare.VNamesEqual(e.Source, src) {
if err := writePartialEdges(ctx, out, assemble.SourceFromEntries(entries)); err != nil {
drainEntries(gsEntries)
return err
}
src = e.Source
entries = nil
}
entries = append(entries, e)
}
if len(entries) > 0 {
if err := writePartialEdges(ctx, out, assemble.SourceFromEntries(entries)); err != nil {
return err
}
}
if err := writeFileTree(ctx, tree, out.xs); err != nil {
return fmt.Errorf("error writing file tree: %v", err)
}
tree = nil
log.Println("Writing complete edges")
snapshot := out.completeEdges.NewSnapshot()
defer snapshot.Close()
it, err := out.completeEdges.ScanPrefix(nil, &keyvalue.Options{
LargeRead: true,
Snapshot: snapshot,
})
if err != nil {
return err
}
defer it.Close()
var n *srvpb.Node
var e srvpb.Edge
for {
k, v, err := it.Next()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("error scanning partial edges table: %v", err)
}
ss := strings.Split(string(k), tempTableKeySep)
if len(ss) != 3 {
return fmt.Errorf("invalid partial edge table key: %q", string(k))
}
if err := proto.Unmarshal(v, &e); err != nil {
return fmt.Errorf("invalid partial edge table value: %v", err)
}
if n == nil || n.Ticket != ss[0] {
n = e.Source
} else if e.Target != nil {
e.Source = n
if err := writeCompletedEdges(ctx, out.completeEdges, &e); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
}
}
return nil
}