func (e *FSTEnum) incr() { e.upto++ e.spi.grow() if len(e.arcs) <= e.upto { newArcs := make([]*Arc, util.Oversize(e.upto+1, util.NUM_BYTES_OBJECT_REF)) copy(newArcs, e.arcs) e.arcs = newArcs } if len(e.output) < e.upto { newOutput := make([]interface{}, util.Oversize(e.upto+1, util.NUM_BYTES_OBJECT_REF)) copy(newOutput, e.output) e.output = newOutput } }
func (br *bytesRef) ensureSize(minSize int) { assert(minSize >= 0) if cap(br.bytes) < minSize { next := make([]byte, util.Oversize(minSize, 1)) copy(next, br.bytes) br.bytes = next } }
func (w *CompressingStoredFieldsWriter) StartDocument(numStoredFields int) error { if w.numBufferedDocs == len(w.numStoredFields) { newLength := util.Oversize(w.numBufferedDocs+1, 4) oldArray := w.endOffsets w.numStoredFields = make([]int, newLength) w.endOffsets = make([]int, newLength) copy(w.numStoredFields, oldArray) copy(w.endOffsets, oldArray) } w.numStoredFields[w.numBufferedDocs] = numStoredFields w.numBufferedDocs++ return nil }
/* Add a new element to this builder. */ func (b *PackedLongValuesBuilderImpl) Add(l int64) PackedLongValuesBuilder { assert2(b.pending != nil, "Cannot be reused after build()") if b.pendingOff == len(b.pending) { // check size if b.valuesOff == len(b.values) { newLength := util.Oversize(b.valuesOff+1, 8) b.grow(newLength) } b.pack() } b.pending[b.pendingOff] = l b.pendingOff++ b.size++ return b }
func (w *CompressingStoredFieldsWriter) FinishDocument() error { if w.numBufferedDocs == len(w.numStoredFields) { newLength := util.Oversize(w.numBufferedDocs+1, 4) oldArray := w.endOffsets w.endOffsets = make([]int, newLength) copy(w.endOffsets, oldArray) oldArray = w.numStoredFields w.numStoredFields = make([]int, newLength) copy(w.numStoredFields, oldArray) } w.numStoredFields[w.numBufferedDocs] = w.numStoredFieldsInDoc w.numStoredFieldsInDoc = 0 w.endOffsets[w.numBufferedDocs] = w.bufferedDocs.length w.numBufferedDocs++ if w.triggerFlush() { return w.flush() } return nil }
func (n *UnCompiledNode) addArc(label int, target Node) { assert(label >= 0) if n.NumArcs != 0 { assert2(label > n.Arcs[n.NumArcs-1].label, "arc[-1].label=%v new label=%v numArcs=%v", n.Arcs[n.NumArcs-1].label, label, n.NumArcs) } if n.NumArcs == len(n.Arcs) { newArcs := make([]*builderArc, util.Oversize(n.NumArcs+1, util.NUM_BYTES_OBJECT_REF)) copy(newArcs, n.Arcs) for arcIdx := n.NumArcs; arcIdx < len(newArcs); arcIdx++ { newArcs[arcIdx] = new(builderArc) } n.Arcs = newArcs } arc := n.Arcs[n.NumArcs] n.NumArcs++ arc.label = label arc.Target = target arc.output = n.owner.NO_OUTPUT arc.nextFinalOutput = n.owner.NO_OUTPUT arc.isFinal = false }
func (a *CharTermAttributeImpl) growTermBuffer(newSize int) { if len(a.termBuffer) < newSize { // not big enough: create a new slice with slight over allocation: a.termBuffer = make([]rune, util.Oversize(newSize, util.NUM_BYTES_CHAR)) } }
func newCharTermAttributeImpl() *CharTermAttributeImpl { return &CharTermAttributeImpl{ termBuffer: make([]rune, util.Oversize(MIN_BUFFER_SIZE, util.NUM_BYTES_CHAR)), bytes: util.NewBytesRefBuilder(), } }
func (arr *ParallelPostingsArray) grow() *ParallelPostingsArray { newSize := util.Oversize(arr.size+1, arr.PostingsArray.bytesPerPosting()) newArray := arr.PostingsArray.newInstance(newSize) arr.PostingsArray.copyTo(newArray, arr.size) return newArray.(*ParallelPostingsArray) }
func newGrowableByteArrayDataOutput(cp int) *GrowableByteArrayDataOutput { ans := &GrowableByteArrayDataOutput{bytes: make([]byte, 0, util.Oversize(cp, 1))} ans.DataOutputImpl = util.NewDataOutput(ans) return ans }
/* It's OK to add the same input twice in a row with different outputs, as long as outputs impls the merge method. Note that input is fully consumed after this method is returned (so caller is free to reuse), but output is not. So if your outputs are changeable (eg ByteSequenceOutputs or IntSequenceOutputs) then you cannot reuse across calls. */ func (b *Builder) Add(input *util.IntsRef, output interface{}) error { // { // debug // bytes := make([]byte, input.Length) // for i, _ := range bytes { // bytes[i] = byte(input.Ints[i]) // } // if output == NO_OUTPUT { // fmt.Printf("\nFST ADD: input=%v %v\n", string(bytes), bytes) // } else { // panic("not implemented yet") // // fmt.Printf("\nFST ADD: input=%v %v output=%v", string(bytes), bytes, b.fst.outputs.outputToString(output))); // } // } // de-dup NO_OUTPUT since it must be a singleton: if output == NO_OUTPUT { output = NO_OUTPUT } assert2(b.lastInput.Length() == 0 || !input.Less(b.lastInput.Get()), "inputs are added out of order, lastInput=%v vs input=%v", b.lastInput.Get(), input) if input.Length == 0 { // empty input: only allowed as first input. We have to special // case this becaues the packed FST format cannot represent the // empty input since 'finalness' is stored on the incoming arc, // not on the node b.frontier[0].InputCount++ b.frontier[0].IsFinal = true b.fst.setEmptyOutput(output) return nil } // compare shared prefix length pos1 := 0 pos2 := input.Offset pos1Stop := b.lastInput.Length() if input.Length < pos1Stop { pos1Stop = input.Length } for { b.frontier[pos1].InputCount++ if pos1 >= pos1Stop || b.lastInput.At(pos1) != input.Ints[pos2] { break } pos1++ pos2++ } prefixLenPlus1 := pos1 + 1 if len(b.frontier) < input.Length+1 { next := make([]*UnCompiledNode, util.Oversize(input.Length+1, util.NUM_BYTES_OBJECT_REF)) copy(next, b.frontier) for idx := len(b.frontier); idx < len(next); idx++ { next[idx] = NewUnCompiledNode(b, idx) } b.frontier = next } // minimize/compile states from previous input's orphan'd suffix err := b.freezeTail(prefixLenPlus1) if err != nil { return err } // init tail states for current input for idx := prefixLenPlus1; idx <= input.Length; idx++ { b.frontier[idx-1].addArc(input.Ints[input.Offset+idx-1], b.frontier[idx]) b.frontier[idx].InputCount++ } lastNode := b.frontier[input.Length] if b.lastInput.Length() != input.Length || prefixLenPlus1 != input.Length+1 { lastNode.IsFinal = true lastNode.output = b.NO_OUTPUT } // push conflicting outputs forward, only as far as needed for idx := 1; idx < prefixLenPlus1; idx++ { node := b.frontier[idx] parentNode := b.frontier[idx-1] lastOutput := parentNode.lastOutput(input.Ints[input.Offset+idx-1]) var commonOutputPrefix interface{} var wordSuffix interface{} if lastOutput != b.NO_OUTPUT { commonOutputPrefix = b.fst.outputs.Common(output, lastOutput) wordSuffix = b.fst.outputs.Subtract(lastOutput, commonOutputPrefix) parentNode.setLastOutput(input.Ints[input.Offset+idx-1], commonOutputPrefix) node.prependOutput(wordSuffix) } else { commonOutputPrefix = NO_OUTPUT } output = b.fst.outputs.Subtract(output, commonOutputPrefix) } if b.lastInput.Length() == input.Length && prefixLenPlus1 == 1+input.Length { // same input more than 1 time in a row, mapping to multiple outputs panic("not implemented yet") } else { // this new arc is private to this new input; set its arc output // to the leftover output: b.frontier[prefixLenPlus1-1].setLastOutput(input.At(prefixLenPlus1-1), output) } // save last input b.lastInput.CopyInts(input) return nil }
func newGrowableByteArrayDataOutput(cp int) *GrowableByteArrayDataOutput { return &GrowableByteArrayDataOutput{make([]byte, 0, util.Oversize(cp, 1)), 0} }