func createFragmentsTable(ctx context.Context, entries <-chan *spb.Entry) (t *table.KVProto, err error) { log.Println("Writing decoration fragments") defer drainEntries(entries) // ensure channel is drained on errors temp, err := tempTable("decor.fragments") if err != nil { return nil, fmt.Errorf("failed to create temporary table: %v", err) } fragments := &table.KVProto{temp} defer func() { if err != nil { if err := fragments.Close(ctx); err != nil { log.Println("Error closing fragments table: %v", err) } } }() for src := range assemble.Sources(entries) { for _, fragment := range assemble.DecorationFragments(src) { fileTicket := fragment.FileTicket var anchorTicket string if len(fragment.Decoration) != 0 { // don't keep decor.FileTicket for anchors; we aren't sure we have a file yet fragment.FileTicket = "" anchorTicket = fragment.Decoration[0].Anchor.Ticket } if err := fragments.Put(ctx, []byte(fileTicket+tempTableKeySep+anchorTicket), fragment); err != nil { return nil, err } } } return fragments, nil }
func combineNodesAndEdges(ctx context.Context, opts *Options, out *servingOutput, rdIn stream.EntryReader) (disksort.Interface, error) { log.Println("Writing partial edges") tree := filetree.NewMap() rd := func(f func(*spb.Entry) error) error { return rdIn(func(e *spb.Entry) error { if e.FactName == facts.NodeKind && string(e.FactValue) == nodes.File { tree.AddFile(e.Source) // TODO(schroederc): evict finished directories (based on GraphStore order) } return f(e) }) } partialSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{}) if err != nil { return nil, err } bIdx := out.idx.Buffered() if err := assemble.Sources(rd, func(src *ipb.Source) error { return writePartialEdges(ctx, partialSorter, bIdx, src) }); err != nil { return nil, err } if err := bIdx.Flush(ctx); err != nil { return nil, err } if err := writeFileTree(ctx, tree, out.xs); err != nil { return nil, fmt.Errorf("error writing file tree: %v", err) } tree = nil log.Println("Writing complete edges") cSorter, err := opts.diskSorter(edgeLesser{}, edgeMarshaler{}) if err != nil { return nil, err } var n *srvpb.Node if err := partialSorter.Read(func(i interface{}) error { e := i.(*srvpb.Edge) if n == nil || n.Ticket != e.Source.Ticket { n = e.Source if e.Target != nil { if opts.Verbose { log.Printf("WARNING: missing node facts for: %q", e.Source.Ticket) } // This is needed to satisfy later parts of the pipeline that look for targetless edges // to signify new nodes. if err := cSorter.Add(&srvpb.Edge{Source: &srvpb.Node{Ticket: e.Source.Ticket}}); err != nil { return fmt.Errorf("error writing complete edge: %v", err) } } } if e.Target == nil { // pass-through self-edges return cSorter.Add(e) } e.Source = n if err := writeCompletedEdges(ctx, cSorter, e); err != nil { return fmt.Errorf("error writing complete edge: %v", err) } return nil }); err != nil { return nil, fmt.Errorf("error reading/writing edges: %v", err) } return cSorter, nil }