コード例 #1
0
ファイル: termsWriter.go プロジェクト: kiskovacs/golucene
func (b *PendingBlock) append(
	builder *fst.Builder,
	subIndex *fst.FST,
	scratchIntsRef *util.IntsRefBuilder) error {

	subIndexEnum := fst.NewBytesRefFSTEnum(subIndex)
	indexEnt, err := subIndexEnum.Next()
	for err == nil && indexEnt != nil {
		// fmt.Printf("      add sub=%v output=%v\n", indexEnt.Input, indexEnt.Output)
		err = builder.Add(fst.ToIntsRef(indexEnt.Input.ToBytes(), scratchIntsRef), indexEnt.Output)
		if err == nil {
			indexEnt, err = subIndexEnum.Next()
		}
	}
	return err
}
コード例 #2
0
ファイル: termsWriter.go プロジェクト: kiskovacs/golucene
func (b *PendingBlock) compileIndex(blocks []*PendingBlock,
	scratchBytes *store.RAMOutputStream,
	scratchIntsRef *util.IntsRefBuilder) (err error) {

	assert2(b.isFloor && len(blocks) > 1 || (!b.isFloor && len(blocks) == 1),
		"isFloor=%v blocks=%v", b.isFloor, blocks)
	assert(blocks[0] == b)

	assert(scratchBytes.FilePointer() == 0)

	// TODO: try writing the leading vLong in MSB order
	// (opposite of what Lucene does today), for better
	// outputs sharing in the FST
	if err = scratchBytes.WriteVLong(encodeOutput(b.fp, b.hasTerms, b.isFloor)); err != nil {
		return
	}
	if b.isFloor {
		if err = scratchBytes.WriteVInt(int32(len(blocks) - 1)); err != nil {
			return
		}
		for _, sub := range blocks[1:] {
			assert(sub.floorLeadByte != -1)
			// fmt.Printf("    write floorLeadByte=%v\n", util.ItoHex(int64(sub.floorLeadByte)))
			if err = scratchBytes.WriteByte(byte(sub.floorLeadByte)); err != nil {
				return
			}
			assert(sub.fp > b.fp)
			if err = scratchBytes.WriteVLong((sub.fp-b.fp)<<1 |
				int64(map[bool]int{true: 1, false: 0}[sub.hasTerms])); err != nil {
				return
			}
		}
	}

	outputs := fst.ByteSequenceOutputsSingleton()
	indexBuilder := fst.NewBuilder(fst.INPUT_TYPE_BYTE1,
		0, 0, true, false, int(math.MaxInt32),
		outputs, false,
		packed.PackedInts.COMPACT, true, 15)

	// fmt.Printf("  compile index for prefix=%v\n", b.prefix)

	bytes := make([]byte, scratchBytes.FilePointer())
	assert(len(bytes) > 0)
	err = scratchBytes.WriteToBytes(bytes)
	if err != nil {
		return err
	}
	err = indexBuilder.Add(fst.ToIntsRef(b.prefix, scratchIntsRef), bytes)
	if err != nil {
		return err
	}
	scratchBytes.Reset()

	// copy over index for all sub-blocks
	for _, block := range blocks {
		if block.subIndices != nil {
			for _, subIndex := range block.subIndices {
				if err = b.append(indexBuilder, subIndex, scratchIntsRef); err != nil {
					return err
				}
			}
		}
		block.subIndices = nil
	}

	if b.index, err = indexBuilder.Finish(); err != nil {
		return err
	}
	assert(b.subIndices == nil)
	return nil
}