Ejemplo n.º 1
0
func (this *FastPFOR) encodePage(in []int32, inpos *cursor.Cursor, thissize int, out []int32, outpos *cursor.Cursor, initoffset *cursor.Cursor) error {
	headerpos := int32(outpos.Get())
	outpos.Increment()
	tmpoutpos := int32(outpos.Get())

	// Clear working area
	copy(this.dataPointers, zeroDataPointers)
	this.byteContainer.Clear()

	tmpinpos := int32(inpos.Get())
	var delta [DefaultBlockSize]int32

	for finalInpos := tmpinpos + int32(thissize) - DefaultBlockSize; tmpinpos <= finalInpos; tmpinpos += DefaultBlockSize {
		// Calculate the deltas, inlining to gain a bit of performance
		offset := int32(initoffset.Get())
		for i, v := range in[tmpinpos : tmpinpos+DefaultBlockSize] {
			n := v - offset
			delta[i] = (n << 1) ^ (n >> 31)
			offset = v
		}

		initoffset.Set(int(in[tmpinpos+DefaultBlockSize-1]))

		//bestb, bestc, maxb := this.getBestBFromData(in[tmpinpos:tmpinpos+DefaultBlockSize])
		bestb, bestc, maxb := this.getBestBFromData(delta[:])
		tmpbestb := bestb
		this.byteContainer.Put(byte(bestb))
		this.byteContainer.Put(byte(bestc))

		if bestc > 0 {
			this.byteContainer.Put(byte(maxb))
			index := maxb - bestb

			if int(this.dataPointers[index]+bestc) >= len(this.dataToBePacked[index]) {
				newSize := int(2 * (this.dataPointers[index] + bestc))

				// make sure it is a multiple of 32.
				// there might be a better way to do this
				newSize = encoding.CeilBy(newSize, 32)
				newSlice := make([]int32, newSize)
				copy(newSlice, this.dataToBePacked[index])
				this.dataToBePacked[index] = newSlice
			}

			for k := int32(0); k < DefaultBlockSize; k++ {
				if uint32(delta[k])>>uint(bestb) != 0 {
					// we have an exception
					this.byteContainer.Put(byte(k))
					this.dataToBePacked[index][this.dataPointers[index]] = int32(uint32(delta[k]) >> uint(tmpbestb))
					this.dataPointers[index] += 1
				}
			}
		}

		for k := int32(0); k < 128; k += 32 {
			bitpacking.FastPack(delta[:], int(k), out, int(tmpoutpos), int(tmpbestb))
			tmpoutpos += tmpbestb
		}
	}

	inpos.Set(int(tmpinpos))
	out[headerpos] = tmpoutpos - headerpos

	for this.byteContainer.Position()&3 != 0 {
		this.byteContainer.Put(0)
	}

	bytesize := int32(this.byteContainer.Position())
	out[tmpoutpos] = bytesize
	tmpoutpos += 1
	howmanyints := bytesize / 4

	this.byteContainer.Flip()
	this.byteContainer.AsInt32Buffer().GetInt32s(out, int(tmpoutpos), int(howmanyints))
	tmpoutpos += howmanyints

	bitmap := int32(0)
	for k := 1; k <= 32; k++ {
		v := this.dataPointers[k]
		if v != 0 {
			bitmap |= (1 << uint(k-1))
		}
	}

	out[tmpoutpos] = bitmap
	tmpoutpos += 1

	for k := 1; k < 33; k++ {
		v := this.dataPointers[k]
		if v != 0 {
			out[tmpoutpos] = v // size
			tmpoutpos += 1
			for j := 0; j < int(v); j += 32 {
				bitpacking.FastPack(this.dataToBePacked[k], j, out, int(tmpoutpos), k)
				tmpoutpos += int32(k)
			}
		}
	}

	outpos.Set(int(tmpoutpos))

	return nil
}
Ejemplo n.º 2
0
func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32, outpos *cursor.Cursor, thissize int, initoffset *cursor.Cursor) error {
	initpos := int32(inpos.Get())
	wheremeta := in[initpos]
	inpos.Increment()

	inexcept := initpos + wheremeta
	bytesize := in[inexcept]
	inexcept += 1

	this.byteContainer.Clear()
	if err := this.byteContainer.AsInt32Buffer().PutInt32s(in, int(inexcept), int(bytesize/4)); err != nil {
		return err
	}

	inexcept += bytesize / 4
	bitmap := in[inexcept]
	inexcept += 1

	for k := int32(1); k < 33; k++ {
		if bitmap&(1<<uint32(k-1)) != 0 {
			size := in[inexcept]
			inexcept += 1

			if int32(len(this.dataToBePacked[k])) < size {
				this.dataToBePacked[k] = make([]int32, encoding.CeilBy(int(size), 32))
			}

			for j := int32(0); j < size; j += 32 {
				bitpacking.FastUnpack(in, int(inexcept), this.dataToBePacked[k], int(j), int(k))
				inexcept += k
			}
		}
	}

	copy(this.dataPointers, zeroDataPointers)
	tmpoutpos := int32(outpos.Get())
	tmpinpos := int32(inpos.Get())

	delta := make([]int32, DefaultBlockSize)

	run := 0
	run_end := thissize / DefaultBlockSize
	for run < run_end {
		var err error
		var bestb int32
		if bestb, err = this.byteContainer.GetAsInt32(); err != nil {
			return err
		}

		var cexcept int32
		if cexcept, err = this.byteContainer.GetAsInt32(); err != nil {
			return err
		}

		for k := int32(0); k < 128; k += 32 {
			//bitpacking.FastUnpack(in, int(tmpinpos), out, int(tmpoutpos+k), int(bestb))
			bitpacking.FastUnpack(in, int(tmpinpos), delta, int(k), int(bestb))
			tmpinpos += bestb
		}

		if cexcept > 0 {
			var maxbits int32
			if maxbits, err = this.byteContainer.GetAsInt32(); err != nil {
				return err
			}

			index := maxbits - bestb

			for k := int32(0); k < cexcept; k++ {
				var pos int32
				if pos, err = this.byteContainer.GetAsInt32(); err != nil {
					return err
				}

				exceptvalue := this.dataToBePacked[index][this.dataPointers[index]]
				this.dataPointers[index] += 1
				//out[pos + tmpoutpos] |= exceptvalue << uint(bestb)
				delta[pos] |= exceptvalue << uint(bestb)
			}
		}

		// Calculate the original from the deltas, inlining to gain a bit of performance
		offset := int32(initoffset.Get())
		for i, v := range delta {
			n := int32(uint32(v)>>1) ^ ((v << 31) >> 31)
			out[int(tmpoutpos)+i] = n + offset
			offset += n
		}
		initoffset.Set(int(out[tmpoutpos+DefaultBlockSize-1]))

		run += 1
		tmpoutpos += DefaultBlockSize
	}

	outpos.Set(int(tmpoutpos))
	inpos.Set(int(inexcept))

	return nil
}
Ejemplo n.º 3
0
func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32, outpos *cursor.Cursor, thissize int) error {
	initpos := int32(inpos.Get())
	wheremeta := in[initpos]
	inpos.Increment()

	inexcept := initpos + wheremeta
	bytesize := in[inexcept]
	inexcept += 1
	mybytearray := in[inexcept:]
	mybp := uint(0)

	inexcept += (bytesize + 3) / 4
	bitmap := in[inexcept]
	inexcept += 1

	for k := int32(1); k < 33; k++ {
		if bitmap&(1<<uint32(k-1)) != 0 {
			size := in[inexcept]
			inexcept += 1

			if int32(len(this.dataToBePacked[k])) < size {
				this.dataToBePacked[k] = make([]int32, encoding.CeilBy(int(size), 32))
			}
			for j := int32(0); j < size; j += 32 {
				bitpacking.FastUnpack(in, int(inexcept), this.dataToBePacked[k], int(j), int(k))
				inexcept += k
			}
		}
	}

	copy(this.dataPointers, zeroDataPointers)
	tmpoutpos := uint32(outpos.Get())
	tmpinpos := uint32(inpos.Get())

	run := 0
	run_end := thissize / DefaultBlockSize
	for run < run_end {
		bestb := uint32(grapByte(mybytearray, mybp))
		mybp++
		cexcept := int32(grapByte(mybytearray, mybp))
		mybp++
		for k := uint32(0); k < 128; k += 32 {
			bitpacking.FastUnpack(in, int(tmpinpos), out, int(tmpoutpos+k), int(bestb))
			tmpinpos += bestb
		}

		if cexcept > 0 {
			maxbits := uint32(grapByte(mybytearray, mybp))
			mybp++
			index := maxbits - bestb
			// assuming that the Go compiler is bad, we move everything that is indexed outside the upcoming loop
			packedexceptions := this.dataToBePacked[index]
			myindex := this.dataPointers[index]

			for k := int32(0); k < cexcept; k++ {
				pos := uint32(grapByte(mybytearray, mybp))
				mybp++
				exceptvalue := packedexceptions[myindex]
				myindex++
				out[pos+tmpoutpos] |= exceptvalue << bestb
			}
			this.dataPointers[index] = myindex
		}

		run += 1
		tmpoutpos += DefaultBlockSize
	}

	outpos.Set(int(tmpoutpos))
	inpos.Set(int(inexcept))

	return nil
}