// Run writes the xrefs and filetree serving tables to db based on the given
// graphstore.Service.
func Run(ctx context.Context, gs graphstore.Service, db keyvalue.DB, opts *Options) error {
if opts == nil {
opts = new(Options)
}
log.Println("Starting serving pipeline")
out := &servingOutput{
xs: table.ProtoBatchParallel{&table.KVProto{DB: db}},
idx: &table.KVInverted{DB: db},
}
entries := make(chan *spb.Entry, chBuf)
var cErr error
var wg sync.WaitGroup
var sortedEdges disksort.Interface
wg.Add(1)
go func() {
sortedEdges, cErr = combineNodesAndEdges(ctx, opts, out, entries)
if cErr != nil {
cErr = fmt.Errorf("error combining nodes and edges: %v", cErr)
}
wg.Done()
}()
err := gs.Scan(ctx, &spb.ScanRequest{}, func(e *spb.Entry) error {
if graphstore.IsNodeFact(e) || schema.EdgeDirection(e.EdgeKind) == schema.Forward {
entries <- e
}
return nil
})
close(entries)
if err != nil {
return fmt.Errorf("error scanning GraphStore: %v", err)
}
wg.Wait()
if cErr != nil {
return cErr
}
pesIn, dIn := make(chan *srvpb.Edge, chBuf), make(chan *srvpb.Edge, chBuf)
var pErr, fErr error
wg.Add(2)
go func() {
defer wg.Done()
if err := writePagedEdges(ctx, pesIn, out.xs, opts); err != nil {
pErr = fmt.Errorf("error writing paged edge sets: %v", err)
}
}()
go func() {
defer wg.Done()
if err := writeDecorAndRefs(ctx, opts, dIn, out); err != nil {
fErr = fmt.Errorf("error writing file decorations: %v", err)
}
}()
err = sortedEdges.Read(func(x interface{}) error {
e := x.(*srvpb.Edge)
pesIn <- e
dIn <- e
return nil
})
close(pesIn)
close(dIn)
if err != nil {
return fmt.Errorf("error reading edges table: %v", err)
}
wg.Wait()
if pErr != nil {
return pErr
}
return fErr
}
// Run writes the xrefs and filetree serving tables to db based on the given
// entries (in GraphStore-order).
func Run(ctx context.Context, rd stream.EntryReader, db keyvalue.DB, opts *Options) error {
if opts == nil {
opts = new(Options)
}
log.Println("Starting serving pipeline")
out := &servingOutput{
xs: table.ProtoBatchParallel{&table.KVProto{DB: db}},
idx: &table.KVInverted{DB: db},
}
rd = filterReverses(rd)
var cErr error
var wg sync.WaitGroup
var sortedEdges disksort.Interface
wg.Add(1)
go func() {
sortedEdges, cErr = combineNodesAndEdges(ctx, opts, out, rd)
if cErr != nil {
cErr = fmt.Errorf("error combining nodes and edges: %v", cErr)
}
wg.Done()
}()
wg.Wait()
if cErr != nil {
return cErr
}
pesIn, dIn := make(chan *srvpb.Edge, chBuf), make(chan *srvpb.Edge, chBuf)
var pErr, fErr error
wg.Add(2)
go func() {
defer wg.Done()
if err := writePagedEdges(ctx, pesIn, out.xs, opts); err != nil {
pErr = fmt.Errorf("error writing paged edge sets: %v", err)
}
}()
go func() {
defer wg.Done()
if err := writeDecorAndRefs(ctx, opts, dIn, out); err != nil {
fErr = fmt.Errorf("error writing file decorations: %v", err)
}
}()
err := sortedEdges.Read(func(x interface{}) error {
e := x.(*srvpb.Edge)
pesIn <- e
dIn <- e
return nil
})
close(pesIn)
close(dIn)
if err != nil {
return fmt.Errorf("error reading edges table: %v", err)
}
wg.Wait()
if pErr != nil {
return pErr
}
return fErr
}