func writeDecorations(ctx context.Context, t table.Proto, entries <-chan *spb.Entry) error {
fragments, err := createFragmentsTable(ctx, entries)
if err != nil {
return fmt.Errorf("error creating fragments table: %v", err)
}
defer fragments.Close(ctx)
it, err := fragments.ScanPrefix(nil, &keyvalue.Options{LargeRead: true})
if err != nil {
return err
}
defer it.Close()
log.Println("Writing Decorations")
var curFile string
var decor *srvpb.FileDecorations
for {
k, v, err := it.Next()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("error scanning temporary table: %v", err)
}
ss := strings.Split(string(k), tempTableKeySep)
if len(ss) != 2 {
return fmt.Errorf("invalid temporary table key: %q", string(k))
}
fileTicket := ss[0]
if decor != nil && curFile != fileTicket {
if decor.FileTicket != "" {
if err := writeDecor(ctx, t, decor); err != nil {
return err
}
}
decor = nil
}
curFile = fileTicket
if decor == nil {
decor = &srvpb.FileDecorations{}
}
var fragment srvpb.FileDecorations
if err := proto.Unmarshal(v, &fragment); err != nil {
return fmt.Errorf("invalid temporary table value: %v", err)
}
if fragment.FileTicket == "" {
decor.Decoration = append(decor.Decoration, fragment.Decoration...)
} else {
decor.FileTicket = fragment.FileTicket
decor.SourceText = fragment.SourceText
decor.Encoding = fragment.Encoding
}
}
if decor != nil && decor.FileTicket != "" {
if err := writeDecor(ctx, t, decor); err != nil {
return err
}
}
return nil
}
func writeDecorAndRefs(ctx context.Context, opts *Options, edges <-chan *srvpb.Edge, out *servingOutput) error {
fragments, err := opts.diskSorter(fragmentLesser{}, fragmentMarshaler{})
if err != nil {
return err
}
log.Println("Writing decoration fragments")
if err := createDecorationFragments(ctx, edges, fragments); err != nil {
return err
}
log.Println("Writing completed FileDecorations")
// refSorter stores a *srvpb.CrossReference for each Decoration from fragments
refSorter, err := opts.diskSorter(refLesser{}, refMarshaler{})
if err != nil {
return fmt.Errorf("error creating sorter: %v", err)
}
buffer := out.xs.Buffered()
var (
curFile string
file *srvpb.File
norm *xrefs.Normalizer
decor *srvpb.FileDecorations
)
if err := fragments.Read(func(x interface{}) error {
df := x.(*decorationFragment)
fileTicket := df.fileTicket
fragment := df.decoration
if decor != nil && curFile != fileTicket {
if decor.File != nil {
if err := writeDecor(ctx, buffer, decor); err != nil {
return err
}
file = nil
}
decor = nil
}
curFile = fileTicket
if decor == nil {
decor = &srvpb.FileDecorations{}
}
if fragment.File == nil {
decor.Decoration = append(decor.Decoration, fragment.Decoration...)
if file == nil {
return errors.New("missing file for anchors")
}
// Reverse each fragment.Decoration to create a *srvpb.CrossReference
for _, d := range fragment.Decoration {
cr, err := assemble.CrossReference(file, norm, d)
if err != nil {
return fmt.Errorf("error assembling cross-reference: %v", err)
}
if err := refSorter.Add(cr); err != nil {
return fmt.Errorf("error adding CrossReference to sorter: %v", err)
}
}
} else {
decor.File = fragment.File
file = fragment.File
norm = xrefs.NewNormalizer(file.Text)
}
return nil
}); err != nil {
return fmt.Errorf("error reading decoration fragments: %v", err)
}
if decor != nil && decor.File != nil {
if err := writeDecor(ctx, buffer, decor); err != nil {
return err
}
}
log.Println("Writing CrossReferences")
xb := &assemble.CrossReferencesBuilder{
MaxPageSize: opts.MaxPageSize,
Output: func(ctx context.Context, s *srvpb.PagedCrossReferences) error {
return buffer.Put(ctx, xsrv.CrossReferencesKey(s.SourceTicket), s)
},
OutputPage: func(ctx context.Context, p *srvpb.PagedCrossReferences_Page) error {
return buffer.Put(ctx, xsrv.CrossReferencesPageKey(p.PageKey), p)
},
}
var curTicket string
if err := refSorter.Read(func(i interface{}) error {
cr := i.(*srvpb.CrossReference)
if curTicket != cr.Referent.Ticket {
curTicket = cr.Referent.Ticket
if err := xb.StartSet(ctx, curTicket); err != nil {
return fmt.Errorf("error starting cross-references set: %v", err)
}
}
g := &srvpb.PagedCrossReferences_Group{
Kind: cr.TargetAnchor.Kind,
Anchor: []*srvpb.ExpandedAnchor{cr.TargetAnchor},
}
if err := xb.AddGroup(ctx, g); err != nil {
return fmt.Errorf("error adding cross-reference: %v", err)
}
return nil
}); err != nil {
return fmt.Errorf("error reading xrefs: %v", err)
}
if err := xb.Flush(ctx); err != nil {
return fmt.Errorf("error flushing cross-references: %v", err)
}
return buffer.Flush(ctx)
}
func writeFileDecorations(ctx context.Context, edges <-chan *srvpb.Edge, out *servingOutput) error {
temp, err := tempTable("decor.fragments")
if err != nil {
return fmt.Errorf("failed to create temporary table: %v", err)
}
fragments := &table.KVProto{temp}
defer func() {
if err := fragments.Close(ctx); err != nil {
log.Printf("Error closing fragments table: %v", err)
}
}()
log.Println("Writing decoration fragments")
if err := createDecorationFragments(ctx, edges, fragments); err != nil {
return err
}
log.Println("Writing completed FileDecorations")
it, err := fragments.ScanPrefix(nil, &keyvalue.Options{LargeRead: true})
if err != nil {
return err
}
defer it.Close()
log.Println("Writing Decorations")
var curFile string
var decor *srvpb.FileDecorations
var fragment srvpb.FileDecorations
for {
k, v, err := it.Next()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("error scanning temporary table: %v", err)
}
ss := strings.Split(string(k), tempTableKeySep)
if len(ss) != 2 {
return fmt.Errorf("invalid temporary table key: %q", string(k))
}
fileTicket := ss[0]
if decor != nil && curFile != fileTicket {
if decor.FileTicket != "" {
if err := writeDecor(ctx, out.xs, decor); err != nil {
return err
}
}
decor = nil
}
curFile = fileTicket
if decor == nil {
decor = &srvpb.FileDecorations{}
}
if err := proto.Unmarshal(v, &fragment); err != nil {
return fmt.Errorf("invalid temporary table value: %v", err)
}
if fragment.FileTicket == "" {
decor.Decoration = append(decor.Decoration, fragment.Decoration...)
} else {
decor.FileTicket = fragment.FileTicket
decor.SourceText = fragment.SourceText
decor.Encoding = fragment.Encoding
}
}
if decor != nil && decor.FileTicket != "" {
if err := writeDecor(ctx, out.xs, decor); err != nil {
return err
}
}
return nil
}