func (b *PendingBlock) append(
builder *fst.Builder,
subIndex *fst.FST,
scratchIntsRef *util.IntsRefBuilder) error {
subIndexEnum := fst.NewBytesRefFSTEnum(subIndex)
indexEnt, err := subIndexEnum.Next()
for err == nil && indexEnt != nil {
// fmt.Printf(" add sub=%v output=%v\n", indexEnt.Input, indexEnt.Output)
err = builder.Add(fst.ToIntsRef(indexEnt.Input.ToBytes(), scratchIntsRef), indexEnt.Output)
if err == nil {
indexEnt, err = subIndexEnum.Next()
}
}
return err
}
func (b *PendingBlock) compileIndex(blocks []*PendingBlock,
scratchBytes *store.RAMOutputStream,
scratchIntsRef *util.IntsRefBuilder) (err error) {
assert2(b.isFloor && len(blocks) > 1 || (!b.isFloor && len(blocks) == 1),
"isFloor=%v blocks=%v", b.isFloor, blocks)
assert(blocks[0] == b)
assert(scratchBytes.FilePointer() == 0)
// TODO: try writing the leading vLong in MSB order
// (opposite of what Lucene does today), for better
// outputs sharing in the FST
if err = scratchBytes.WriteVLong(encodeOutput(b.fp, b.hasTerms, b.isFloor)); err != nil {
return
}
if b.isFloor {
if err = scratchBytes.WriteVInt(int32(len(blocks) - 1)); err != nil {
return
}
for _, sub := range blocks[1:] {
assert(sub.floorLeadByte != -1)
// fmt.Printf(" write floorLeadByte=%v\n", util.ItoHex(int64(sub.floorLeadByte)))
if err = scratchBytes.WriteByte(byte(sub.floorLeadByte)); err != nil {
return
}
assert(sub.fp > b.fp)
if err = scratchBytes.WriteVLong((sub.fp-b.fp)<<1 |
int64(map[bool]int{true: 1, false: 0}[sub.hasTerms])); err != nil {
return
}
}
}
outputs := fst.ByteSequenceOutputsSingleton()
indexBuilder := fst.NewBuilder(fst.INPUT_TYPE_BYTE1,
0, 0, true, false, int(math.MaxInt32),
outputs, false,
packed.PackedInts.COMPACT, true, 15)
// fmt.Printf(" compile index for prefix=%v\n", b.prefix)
bytes := make([]byte, scratchBytes.FilePointer())
assert(len(bytes) > 0)
err = scratchBytes.WriteToBytes(bytes)
if err != nil {
return err
}
err = indexBuilder.Add(fst.ToIntsRef(b.prefix, scratchIntsRef), bytes)
if err != nil {
return err
}
scratchBytes.Reset()
// copy over index for all sub-blocks
for _, block := range blocks {
if block.subIndices != nil {
for _, subIndex := range block.subIndices {
if err = b.append(indexBuilder, subIndex, scratchIntsRef); err != nil {
return err
}
}
}
block.subIndices = nil
}
if b.index, err = indexBuilder.Finish(); err != nil {
return err
}
assert(b.subIndices == nil)
return nil
}