func (this *FastPFOR) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
inlength = encoding.FloorBy(inlength, DefaultBlockSize)
if inlength == 0 {
return errors.New("fastpfor/Compress: inlength = 0. No work done.")
}
out[outpos.Get()] = int32(inlength)
outpos.Increment()
initoffset := cursor.New()
copy(this.dataPointers, zeroDataPointers)
copy(this.freqs, zeroFreqs)
finalInpos := inpos.Get() + inlength
for inpos.Get() != finalInpos {
thissize := int(math.Min(float64(this.pageSize), float64(finalInpos-inpos.Get())))
if err := this.encodePage(in, inpos, thissize, out, outpos, initoffset); err != nil {
return errors.New("fastpfor/Compress: " + err.Error())
}
}
return nil
}
func (this *BP32) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("zigzag_bp32/Uncompress: Length is 0. No work done.")
}
outlength := int(in[inpos.Get()])
inpos.Increment()
tmpinpos := inpos.Get()
s := outpos.Get()
finalinpos := s + outlength
delta := make([]int32, DefaultBlockSize)
//log.Printf("zigzag_bp32/Uncompress: outlength = %d, inpos = %d, outpos = %d\n", outlength, inpos.Get(), outpos.Get())
for ; s < finalinpos; s += DefaultBlockSize {
tmp := in[tmpinpos]
mbits1 := tmp >> 24
mbits2 := (tmp >> 16) & 0xFF
mbits3 := (tmp >> 8) & 0xFF
mbits4 := (tmp) & 0xFF
//log.Printf("zigzag_bp32/Uncopmress: mbits1 = %d, mbits2 = %d, mbits3 = %d, mbits4 = %d, s = %d\n", mbits1, mbits2, mbits3, mbits4, s)
tmpinpos += 1
bitpacking.FastUnpack(in, tmpinpos, delta, 0, int(mbits1))
tmpinpos += int(mbits1)
//log.Printf("zigzag_bp32/Uncompress: delta = %v\n", out)
bitpacking.FastUnpack(in, tmpinpos, delta, 32, int(mbits2))
tmpinpos += int(mbits2)
//log.Printf("zigzag_bp32/Uncompress: delta = %v\n", out)
bitpacking.FastUnpack(in, tmpinpos, delta, 64, int(mbits3))
tmpinpos += int(mbits3)
//log.Printf("zigzag_bp32/Uncompress: delta = %v\n", out)
bitpacking.FastUnpack(in, tmpinpos, delta, 96, int(mbits4))
tmpinpos += int(mbits4)
encoding.InverseZigZagDelta(delta, out[s:s+DefaultBlockSize])
//log.Printf("zigzag_bp32/Uncompress: delta = %v\n", delta)
//log.Printf("zigzag_bp32/Uncompress: out = %v\n", out[s:s+DefaultBlockSize])
}
outpos.Add(outlength)
inpos.Set(tmpinpos)
return nil
}
func (this *BP32) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("BP32/Uncompress: Length is 0. No work done.")
}
outlength := in[inpos.Get()]
inpos.Increment()
tmpinpos := inpos.Get()
initoffset := int32(0)
//log.Printf("bp32/Uncompress: outlength = %d, inpos = %d, outpos = %d\n", outlength, inpos.Get(), outpos.Get())
for s := outpos.Get(); s < outpos.Get()+int(outlength); s += 32 * 4 {
tmp := in[tmpinpos]
mbits1 := tmp >> 24
mbits2 := (tmp >> 16) & 0xFF
mbits3 := (tmp >> 8) & 0xFF
mbits4 := (tmp) & 0xFF
//log.Printf("bp32/Uncopmress: mbits1 = %d, mbits2 = %d, mbits3 = %d, mbits4 = %d, s = %d\n", mbits1, mbits2, mbits3, mbits4, s)
tmpinpos += 1
bitpacking.DeltaUnpack(initoffset, in, tmpinpos, out, s, int(mbits1))
tmpinpos += int(mbits1)
initoffset = out[s+31]
//log.Printf("bp32/Uncompress: out = %v\n", out)
bitpacking.DeltaUnpack(initoffset, in, tmpinpos, out, s+32, int(mbits2))
tmpinpos += int(mbits2)
initoffset = out[s+32+31]
//log.Printf("bp32/Uncompress: out = %v\n", out)
bitpacking.DeltaUnpack(initoffset, in, tmpinpos, out, s+2*32, int(mbits3))
tmpinpos += int(mbits3)
initoffset = out[s+2*32+31]
//log.Printf("bp32/Uncompress: out = %v\n", out)
bitpacking.DeltaUnpack(initoffset, in, tmpinpos, out, s+3*32, int(mbits4))
tmpinpos += int(mbits4)
initoffset = out[s+3*32+31]
//log.Printf("bp32/Uncompress: out = %v\n", out)
}
outpos.Add(int(outlength))
inpos.Set(tmpinpos)
return nil
}
func (this *FastPFOR) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("fastpfor/Uncompress: inlength = 0. No work done.")
}
mynvalue := in[inpos.Get()]
inpos.Increment()
copy(this.dataPointers, zeroDataPointers)
finalout := outpos.Get() + int(mynvalue)
for outpos.Get() != finalout {
thissize := int(math.Min(float64(this.pageSize), float64(finalout-outpos.Get())))
if err := this.decodePage(in, inpos, out, outpos, thissize); err != nil {
return errors.New("fastpfor/Uncompress: " + err.Error())
}
}
return nil
}
func (this *Composition) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("composition/Compress: inlength = 0. No work done.")
}
init := inpos.Get()
this.f1.Compress(in, inpos, inlength, out, outpos)
if outpos.Get() == 0 {
out[0] = 0
outpos.Increment()
}
//log.Printf("composition/Compress: f1 inpos = %d, outpos = %d, inlength = %d\n", inpos.Get(), outpos.Get(), inlength)
inlength -= inpos.Get() - init
this.f2.Compress(in, inpos, inlength, out, outpos)
//log.Printf("composition/Compress: f2 inpos = %d, outpos = %d, inlength = %d\n", inpos.Get(), outpos.Get(), inlength)
return nil
}
func (this *BP32) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
inlength = encoding.FloorBy(inlength, DefaultBlockSize)
if inlength == 0 {
return errors.New("BP32/Compress: block size less than 128. No work done.")
}
out[outpos.Get()] = int32(inlength)
outpos.Increment()
tmpoutpos := outpos.Get()
s := inpos.Get()
finalinpos := s + inlength
for ; s < finalinpos; s += DefaultBlockSize {
mbits1 := encoding.MaxBits(in[s : s+32])
mbits2 := encoding.MaxBits(in[s+32 : s+2*32])
mbits3 := encoding.MaxBits(in[s+2*32 : s+3*32])
mbits4 := encoding.MaxBits(in[s+3*32 : s+4*32])
out[tmpoutpos] = (mbits1 << 24) | (mbits2 << 16) | (mbits3 << 8) | mbits4
tmpoutpos += 1
bitpacking.FastPackWithoutMask(in, s, out, tmpoutpos, int(mbits1))
tmpoutpos += int(mbits1)
bitpacking.FastPackWithoutMask(in, s+32, out, tmpoutpos, int(mbits2))
tmpoutpos += int(mbits2)
bitpacking.FastPackWithoutMask(in, s+2*32, out, tmpoutpos, int(mbits3))
tmpoutpos += int(mbits3)
bitpacking.FastPackWithoutMask(in, s+3*32, out, tmpoutpos, int(mbits4))
tmpoutpos += int(mbits4)
}
inpos.Add(inlength)
outpos.Set(tmpoutpos)
return nil
}
func (this *BP32) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("BP32/Uncompress: Length is 0. No work done.")
}
outlength := int(in[inpos.Get()])
inpos.Increment()
tmpinpos := inpos.Get()
for s := outpos.Get(); s < outpos.Get()+outlength; s += 32 * 4 {
tmp := in[tmpinpos]
mbits1 := tmp >> 24
mbits2 := (tmp >> 16) & 0xFF
mbits3 := (tmp >> 8) & 0xFF
mbits4 := (tmp) & 0xFF
tmpinpos += 1
bitpacking.FastUnpack(in, tmpinpos, out, s, int(mbits1))
tmpinpos += int(mbits1)
bitpacking.FastUnpack(in, tmpinpos, out, s+32, int(mbits2))
tmpinpos += int(mbits2)
bitpacking.FastUnpack(in, tmpinpos, out, s+2*32, int(mbits3))
tmpinpos += int(mbits3)
bitpacking.FastUnpack(in, tmpinpos, out, s+3*32, int(mbits4))
tmpinpos += int(mbits4)
}
outpos.Add(outlength)
inpos.Set(tmpinpos)
return nil
}
func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32, outpos *cursor.Cursor, thissize int, initoffset *cursor.Cursor) error {
initpos := int32(inpos.Get())
wheremeta := in[initpos]
inpos.Increment()
inexcept := initpos + wheremeta
bytesize := in[inexcept]
inexcept += 1
this.byteContainer.Clear()
if err := this.byteContainer.AsInt32Buffer().PutInt32s(in, int(inexcept), int(bytesize/4)); err != nil {
return err
}
inexcept += bytesize / 4
bitmap := in[inexcept]
inexcept += 1
for k := int32(1); k < 33; k++ {
if bitmap&(1<<uint32(k-1)) != 0 {
size := in[inexcept]
inexcept += 1
if int32(len(this.dataToBePacked[k])) < size {
this.dataToBePacked[k] = make([]int32, encoding.CeilBy(int(size), 32))
}
for j := int32(0); j < size; j += 32 {
bitpacking.FastUnpack(in, int(inexcept), this.dataToBePacked[k], int(j), int(k))
inexcept += k
}
}
}
copy(this.dataPointers, zeroDataPointers)
tmpoutpos := int32(outpos.Get())
tmpinpos := int32(inpos.Get())
delta := make([]int32, DefaultBlockSize)
run := 0
run_end := thissize / DefaultBlockSize
for run < run_end {
var err error
var bestb int32
if bestb, err = this.byteContainer.GetAsInt32(); err != nil {
return err
}
var cexcept int32
if cexcept, err = this.byteContainer.GetAsInt32(); err != nil {
return err
}
for k := int32(0); k < 128; k += 32 {
//bitpacking.FastUnpack(in, int(tmpinpos), out, int(tmpoutpos+k), int(bestb))
bitpacking.FastUnpack(in, int(tmpinpos), delta, int(k), int(bestb))
tmpinpos += bestb
}
if cexcept > 0 {
var maxbits int32
if maxbits, err = this.byteContainer.GetAsInt32(); err != nil {
return err
}
index := maxbits - bestb
for k := int32(0); k < cexcept; k++ {
var pos int32
if pos, err = this.byteContainer.GetAsInt32(); err != nil {
return err
}
exceptvalue := this.dataToBePacked[index][this.dataPointers[index]]
this.dataPointers[index] += 1
//out[pos + tmpoutpos] |= exceptvalue << uint(bestb)
delta[pos] |= exceptvalue << uint(bestb)
}
}
// Calculate the original from the deltas, inlining to gain a bit of performance
offset := int32(initoffset.Get())
for i, v := range delta {
n := int32(uint32(v)>>1) ^ ((v << 31) >> 31)
out[int(tmpoutpos)+i] = n + offset
offset += n
}
initoffset.Set(int(out[tmpoutpos+DefaultBlockSize-1]))
run += 1
tmpoutpos += DefaultBlockSize
}
outpos.Set(int(tmpoutpos))
inpos.Set(int(inexcept))
return nil
}
func (this *FastPFOR) encodePage(in []int32, inpos *cursor.Cursor, thissize int, out []int32, outpos *cursor.Cursor, initoffset *cursor.Cursor) error {
headerpos := int32(outpos.Get())
outpos.Increment()
tmpoutpos := int32(outpos.Get())
// Clear working area
copy(this.dataPointers, zeroDataPointers)
this.byteContainer.Clear()
tmpinpos := int32(inpos.Get())
var delta [DefaultBlockSize]int32
for finalInpos := tmpinpos + int32(thissize) - DefaultBlockSize; tmpinpos <= finalInpos; tmpinpos += DefaultBlockSize {
// Calculate the deltas, inlining to gain a bit of performance
offset := int32(initoffset.Get())
for i, v := range in[tmpinpos : tmpinpos+DefaultBlockSize] {
n := v - offset
delta[i] = (n << 1) ^ (n >> 31)
offset = v
}
initoffset.Set(int(in[tmpinpos+DefaultBlockSize-1]))
//bestb, bestc, maxb := this.getBestBFromData(in[tmpinpos:tmpinpos+DefaultBlockSize])
bestb, bestc, maxb := this.getBestBFromData(delta[:])
tmpbestb := bestb
this.byteContainer.Put(byte(bestb))
this.byteContainer.Put(byte(bestc))
if bestc > 0 {
this.byteContainer.Put(byte(maxb))
index := maxb - bestb
if int(this.dataPointers[index]+bestc) >= len(this.dataToBePacked[index]) {
newSize := int(2 * (this.dataPointers[index] + bestc))
// make sure it is a multiple of 32.
// there might be a better way to do this
newSize = encoding.CeilBy(newSize, 32)
newSlice := make([]int32, newSize)
copy(newSlice, this.dataToBePacked[index])
this.dataToBePacked[index] = newSlice
}
for k := int32(0); k < DefaultBlockSize; k++ {
if uint32(delta[k])>>uint(bestb) != 0 {
// we have an exception
this.byteContainer.Put(byte(k))
this.dataToBePacked[index][this.dataPointers[index]] = int32(uint32(delta[k]) >> uint(tmpbestb))
this.dataPointers[index] += 1
}
}
}
for k := int32(0); k < 128; k += 32 {
bitpacking.FastPack(delta[:], int(k), out, int(tmpoutpos), int(tmpbestb))
tmpoutpos += tmpbestb
}
}
inpos.Set(int(tmpinpos))
out[headerpos] = tmpoutpos - headerpos
for this.byteContainer.Position()&3 != 0 {
this.byteContainer.Put(0)
}
bytesize := int32(this.byteContainer.Position())
out[tmpoutpos] = bytesize
tmpoutpos += 1
howmanyints := bytesize / 4
this.byteContainer.Flip()
this.byteContainer.AsInt32Buffer().GetInt32s(out, int(tmpoutpos), int(howmanyints))
tmpoutpos += howmanyints
bitmap := int32(0)
for k := 1; k <= 32; k++ {
v := this.dataPointers[k]
if v != 0 {
bitmap |= (1 << uint(k-1))
}
}
out[tmpoutpos] = bitmap
tmpoutpos += 1
for k := 1; k < 33; k++ {
v := this.dataPointers[k]
if v != 0 {
out[tmpoutpos] = v // size
tmpoutpos += 1
for j := 0; j < int(v); j += 32 {
bitpacking.FastPack(this.dataToBePacked[k], j, out, int(tmpoutpos), k)
tmpoutpos += int32(k)
}
}
}
outpos.Set(int(tmpoutpos))
return nil
}
func (this *BP32) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
//log.Printf("bp32/Compress: before inlength = %d\n", inlength)
inlength = encoding.FloorBy(inlength, DefaultBlockSize)
if inlength == 0 {
return errors.New("BP32/Compress: block size less than 128. No work done.")
}
//log.Printf("bp32/Compress: after inlength = %d, len(in) = %d\n", inlength, len(in))
out[outpos.Get()] = int32(inlength)
outpos.Increment()
tmpoutpos := outpos.Get()
initoffset := int32(0)
s := inpos.Get()
finalinpos := s + inlength
for ; s < finalinpos; s += DefaultBlockSize {
mbits1 := encoding.DeltaMaxBits(initoffset, in[s:s+32])
initoffset2 := in[s+31]
mbits2 := encoding.DeltaMaxBits(initoffset2, in[s+32:s+2*32])
initoffset3 := in[s+32+31]
mbits3 := encoding.DeltaMaxBits(initoffset3, in[s+2*32:s+3*32])
initoffset4 := in[s+2*32+31]
mbits4 := encoding.DeltaMaxBits(initoffset4, in[s+3*32:s+4*32])
//log.Printf("bp32/Compress: tmpoutpos = %d, s = %d\n", tmpoutpos, s)
out[tmpoutpos] = (mbits1 << 24) | (mbits2 << 16) | (mbits3 << 8) | mbits4
tmpoutpos += 1
//log.Printf("bp32/Compress: mbits1 = %d, mbits2 = %d, mbits3 = %d, mbits4 = %d, s = %d\n", mbits1, mbits2, mbits3, mbits4, out[tmpoutpos-1])
bitpacking.DeltaPack(initoffset, in, s, out, tmpoutpos, int(mbits1))
//encoding.PrintUint32sInBits(in, s, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits1))
tmpoutpos += int(mbits1)
bitpacking.DeltaPack(initoffset2, in, s+32, out, tmpoutpos, int(mbits2))
//encoding.PrintUint32sInBits(in, s+32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits2))
tmpoutpos += int(mbits2)
bitpacking.DeltaPack(initoffset3, in, s+2*32, out, tmpoutpos, int(mbits3))
//encoding.PrintUint32sInBits(in, s+2*32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits3))
tmpoutpos += int(mbits3)
bitpacking.DeltaPack(initoffset4, in, s+3*32, out, tmpoutpos, int(mbits4))
//encoding.PrintUint32sInBits(in, s+3*32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits4))
tmpoutpos += int(mbits4)
initoffset = in[s+3*32+31]
}
inpos.Add(inlength)
outpos.Set(tmpoutpos)
return nil
}
func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32, outpos *cursor.Cursor, thissize int) error {
initpos := int32(inpos.Get())
wheremeta := in[initpos]
inpos.Increment()
inexcept := initpos + wheremeta
bytesize := in[inexcept]
inexcept += 1
mybytearray := in[inexcept:]
mybp := uint(0)
inexcept += (bytesize + 3) / 4
bitmap := in[inexcept]
inexcept += 1
for k := int32(1); k < 33; k++ {
if bitmap&(1<<uint32(k-1)) != 0 {
size := in[inexcept]
inexcept += 1
if int32(len(this.dataToBePacked[k])) < size {
this.dataToBePacked[k] = make([]int32, encoding.CeilBy(int(size), 32))
}
for j := int32(0); j < size; j += 32 {
bitpacking.FastUnpack(in, int(inexcept), this.dataToBePacked[k], int(j), int(k))
inexcept += k
}
}
}
copy(this.dataPointers, zeroDataPointers)
tmpoutpos := uint32(outpos.Get())
tmpinpos := uint32(inpos.Get())
run := 0
run_end := thissize / DefaultBlockSize
for run < run_end {
bestb := uint32(grapByte(mybytearray, mybp))
mybp++
cexcept := int32(grapByte(mybytearray, mybp))
mybp++
for k := uint32(0); k < 128; k += 32 {
bitpacking.FastUnpack(in, int(tmpinpos), out, int(tmpoutpos+k), int(bestb))
tmpinpos += bestb
}
if cexcept > 0 {
maxbits := uint32(grapByte(mybytearray, mybp))
mybp++
index := maxbits - bestb
// assuming that the Go compiler is bad, we move everything that is indexed outside the upcoming loop
packedexceptions := this.dataToBePacked[index]
myindex := this.dataPointers[index]
for k := int32(0); k < cexcept; k++ {
pos := uint32(grapByte(mybytearray, mybp))
mybp++
exceptvalue := packedexceptions[myindex]
myindex++
out[pos+tmpoutpos] |= exceptvalue << bestb
}
this.dataPointers[index] = myindex
}
run += 1
tmpoutpos += DefaultBlockSize
}
outpos.Set(int(tmpoutpos))
inpos.Set(int(inexcept))
return nil
}
func (this *BP32) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
//log.Printf("zigzag_bp32/Compress: before inlength = %d\n", inlength)
inlength = encoding.FloorBy(inlength, DefaultBlockSize)
if inlength == 0 {
return errors.New("zigzag_bp32/Compress: block size less than 128. No work done.")
}
//log.Printf("zigzag_bp32/Compress: after inlength = %d, len(in) = %d\n", inlength, len(in))
out[outpos.Get()] = int32(inlength)
outpos.Increment()
tmpoutpos := outpos.Get()
s := inpos.Get()
finalinpos := s + inlength
delta := make([]int32, DefaultBlockSize)
for ; s < finalinpos; s += DefaultBlockSize {
encoding.ZigZagDelta(in[s:s+DefaultBlockSize], delta)
//log.Printf("zigzag_bp32/Compress: in = %v\n", in[s:s+DefaultBlockSize])
//log.Printf("zigzag_bp32/Compress: delta = %v\n", delta)
mbits1 := encoding.MaxBits(delta[0:32])
mbits2 := encoding.MaxBits(delta[32:64])
mbits3 := encoding.MaxBits(delta[64:96])
mbits4 := encoding.MaxBits(delta[96:128])
//log.Printf("zigzag_bp32/Compress: tmpoutpos = %d, s = %d\n", tmpoutpos, s)
out[tmpoutpos] = (mbits1 << 24) | (mbits2 << 16) | (mbits3 << 8) | mbits4
tmpoutpos += 1
//log.Printf("zigzag_bp32/Compress: mbits1 = %d, mbits2 = %d, mbits3 = %d, mbits4 = %d, s = %d\n", mbits1, mbits2, mbits3, mbits4, out[tmpoutpos-1])
bitpacking.FastPackWithoutMask(delta, 0, out, tmpoutpos, int(mbits1))
//encoding.PrintUint32sInBits(in[s:s+32])
//encoding.PrintUint32sInBits(out[tmpoutpos:tmpoutpos+int(mbits1]))
tmpoutpos += int(mbits1)
bitpacking.FastPackWithoutMask(delta, 32, out, tmpoutpos, int(mbits2))
//encoding.PrintUint32sInBits(in, s+32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits2))
tmpoutpos += int(mbits2)
bitpacking.FastPackWithoutMask(delta, 64, out, tmpoutpos, int(mbits3))
//encoding.PrintUint32sInBits(in, s+2*32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits3))
tmpoutpos += int(mbits3)
bitpacking.FastPackWithoutMask(delta, 96, out, tmpoutpos, int(mbits4))
//encoding.PrintUint32sInBits(in, s+3*32, 32)
//encoding.PrintUint32sInBits(out, tmpoutpos, int(mbits4))
tmpoutpos += int(mbits4)
//log.Printf("zigzag_bp32/Compress: out = %v\n", out[s:s+DefaultBlockSize])
}
inpos.Add(inlength)
outpos.Set(tmpoutpos)
return nil
}