func createFragmentsTable(ctx context.Context, entries <-chan *spb.Entry) (t *table.KVProto, err error) {
log.Println("Writing decoration fragments")
defer drainEntries(entries) // ensure channel is drained on errors
temp, err := tempTable("decor.fragments")
if err != nil {
return nil, fmt.Errorf("failed to create temporary table: %v", err)
}
fragments := &table.KVProto{temp}
defer func() {
if err != nil {
if err := fragments.Close(ctx); err != nil {
log.Println("Error closing fragments table: %v", err)
}
}
}()
for src := range assemble.Sources(entries) {
for _, fragment := range assemble.DecorationFragments(src) {
fileTicket := fragment.FileTicket
var anchorTicket string
if len(fragment.Decoration) != 0 {
// don't keep decor.FileTicket for anchors; we aren't sure we have a file yet
fragment.FileTicket = ""
anchorTicket = fragment.Decoration[0].Anchor.Ticket
}
if err := fragments.Put(ctx, []byte(fileTicket+tempTableKeySep+anchorTicket), fragment); err != nil {
return nil, err
}
}
}
return fragments, nil
}
func combineNodesAndEdges(ctx context.Context, opts *Options, out *servingOutput, rdIn stream.EntryReader) (disksort.Interface, error) {
log.Println("Writing partial edges")
tree := filetree.NewMap()
rd := func(f func(*spb.Entry) error) error {
return rdIn(func(e *spb.Entry) error {
if e.FactName == facts.NodeKind && string(e.FactValue) == nodes.File {
tree.AddFile(e.Source)
// TODO(schroederc): evict finished directories (based on GraphStore order)
}
return f(e)
})
}
partialSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
if err != nil {
return nil, err
}
bIdx := out.idx.Buffered()
if err := assemble.Sources(rd, func(src *ipb.Source) error {
return writePartialEdges(ctx, partialSorter, bIdx, src)
}); err != nil {
return nil, err
}
if err := bIdx.Flush(ctx); err != nil {
return nil, err
}
if err := writeFileTree(ctx, tree, out.xs); err != nil {
return nil, fmt.Errorf("error writing file tree: %v", err)
}
tree = nil
log.Println("Writing complete edges")
cSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{})
if err != nil {
return nil, err
}
var n *srvpb.Node
if err := partialSorter.Read(func(i interface{}) error {
e := i.(*srvpb.Edge)
if n == nil || n.Ticket != e.Source.Ticket {
n = e.Source
if e.Target != nil {
if opts.Verbose {
log.Printf("WARNING: missing node facts for: %q", e.Source.Ticket)
}
// This is needed to satisfy later parts of the pipeline that look for targetless edges
// to signify new nodes.
if err := cSorter.Add(&srvpb.Edge{Source: &srvpb.Node{Ticket: e.Source.Ticket}}); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
}
}
if e.Target == nil {
// pass-through self-edges
return cSorter.Add(e)
}
e.Source = n
if err := writeCompletedEdges(ctx, cSorter, e); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
return nil
}); err != nil {
return nil, fmt.Errorf("error reading/writing edges: %v", err)
}
return cSorter, nil
}