// SplitSortedKeyValues constructs a SplitInput that returns a Input
// for each set of *ipb.SortedKeyValues with the same key.
func SplitSortedKeyValues(sorter disksort.Interface) (SplitInput, error) {
iter, err := sorter.Iterator()
if err != nil {
return nil, err
}
return &sortSplitInput{iter: iter}, nil
}
func writePartialEdges(ctx context.Context, sorter disksort.Interface, idx table.BufferedInverted, src *ipb.Source) error {
edges := assemble.PartialReverseEdges(src)
for _, pe := range edges {
if err := sorter.Add(pe); err != nil {
return err
}
}
return nil
}
func createDecorationFragments(ctx context.Context, edges <-chan *srvpb.Edge, fragments disksort.Interface) error {
fdb := &assemble.DecorationFragmentBuilder{
Output: func(ctx context.Context, file string, fragment *srvpb.FileDecorations) error {
return fragments.Add(&decorationFragment{fileTicket: file, decoration: fragment})
},
}
for e := range edges {
if err := fdb.AddEdge(ctx, e); err != nil {
for range edges { // drain input channel
}
return err
}
}
return fdb.Flush(ctx)
}
func writeCompletedEdges(ctx context.Context, edges disksort.Interface, e *srvpb.Edge) error {
if err := edges.Add(&srvpb.Edge{
Source: &srvpb.Node{Ticket: e.Source.Ticket},
Kind: e.Kind,
Target: e.Target,
}); err != nil {
return fmt.Errorf("error writing complete edge: %v", err)
}
if err := edges.Add(&srvpb.Edge{
Source: &srvpb.Node{Ticket: e.Target.Ticket},
Kind: schema.MirrorEdge(e.Kind),
Target: assemble.FilterTextFacts(e.Source),
}); err != nil {
return fmt.Errorf("error writing complete edge mirror: %v", err)
}
return nil
}
// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB, opts *Options) error {
if opts == nil {
opts = new(Options)
}
log.Println("Starting serving pipeline")
out := &servingOutput{
xs: table.ProtoBatchParallel{&table.KVProto{DB: db}},
idx: &table.KVInverted{DB: db},
}
entries := make(chan *spb.Entry, chBuf)
var cErr error
var wg sync.WaitGroup
var sortedEdges disksort.Interface
wg.Add(1)
go func() {
sortedEdges, cErr = combineNodesAndEdges(ctx, opts, out, entries)
if cErr != nil {
cErr = fmt.Errorf("error combining nodes and edges: %v", cErr)
}
wg.Done()
}()
err := gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
if graphstore.IsNodeFact(e) || schema.EdgeDirection(e.EdgeKind) == schema.Forward {
entries <- e
}
return nil
})
close(entries)
if err != nil {
return fmt.Errorf("error scanning GraphStore: %v", err)
}
wg.Wait()
if cErr != nil {
return cErr
}
pesIn, dIn := make(chan *srvpb.Edge, chBuf), make(chan *srvpb.Edge, chBuf)
var pErr, fErr error
wg.Add(2)
go func() {
defer wg.Done()
if err := writePagedEdges(ctx, pesIn, out.xs, opts); err != nil {
pErr = fmt.Errorf("error writing paged edge sets: %v", err)
}
}()
go func() {
defer wg.Done()
if err := writeDecorAndRefs(ctx, opts, dIn, out); err != nil {
fErr = fmt.Errorf("error writing file decorations: %v", err)
}
}()
err = sortedEdges.Read(func(x interface{}) error {
e := x.(*srvpb.Edge)
pesIn <- e
dIn <- e
return nil
})
close(pesIn)
close(dIn)
if err != nil {
return fmt.Errorf("error reading edges table: %v", err)
}
wg.Wait()
if pErr != nil {
return pErr
}
return fErr
}
// Run writes the xrefs and filetree serving tables to db based on the given
// entries (in GraphStore-order).
func Run(ctx context.Context, rd stream.EntryReader, db keyvalue.DB, opts *Options) error {
if opts == nil {
opts = new(Options)
}
log.Println("Starting serving pipeline")
out := &servingOutput{
xs: table.ProtoBatchParallel{&table.KVProto{DB: db}},
idx: &table.KVInverted{DB: db},
}
rd = filterReverses(rd)
var cErr error
var wg sync.WaitGroup
var sortedEdges disksort.Interface
wg.Add(1)
go func() {
sortedEdges, cErr = combineNodesAndEdges(ctx, opts, out, rd)
if cErr != nil {
cErr = fmt.Errorf("error combining nodes and edges: %v", cErr)
}
wg.Done()
}()
wg.Wait()
if cErr != nil {
return cErr
}
pesIn, dIn := make(chan *srvpb.Edge, chBuf), make(chan *srvpb.Edge, chBuf)
var pErr, fErr error
wg.Add(2)
go func() {
defer wg.Done()
if err := writePagedEdges(ctx, pesIn, out.xs, opts); err != nil {
pErr = fmt.Errorf("error writing paged edge sets: %v", err)
}
}()
go func() {
defer wg.Done()
if err := writeDecorAndRefs(ctx, opts, dIn, out); err != nil {
fErr = fmt.Errorf("error writing file decorations: %v", err)
}
}()
err := sortedEdges.Read(func(x interface{}) error {
e := x.(*srvpb.Edge)
pesIn <- e
dIn <- e
return nil
})
close(pesIn)
close(dIn)
if err != nil {
return fmt.Errorf("error reading edges table: %v", err)
}
wg.Wait()
if pErr != nil {
return pErr
}
return fErr
}