func RunCompress(codec encoding.Integer, in []int32, length int, prof bool) (duration int64, out []int32, err error) {
out = make([]int32, length*2)
inpos := cursor.New()
outpos := cursor.New()
now := time.Now()
if prof {
f, e := os.Create("cpu.compress.pprof")
if e != nil {
log.Fatal(e)
}
defer f.Close()
pprof.StartCPUProfile(f)
}
if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
return 0, nil, err
}
since := time.Since(now).Nanoseconds()
if prof {
pprof.StopCPUProfile()
}
return since, out[:outpos.Get()], nil
}
// Compress compresses in[]int32 to out[]int32
func Compress32(in []int32) (out []int32, err error) {
out = make([]int32, len(in)*2)
inpos := cursor.New()
outpos := cursor.New()
if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
return nil, err
}
return out[:outpos.Get()], nil
}
// Uncompress uncompresses in[]int32 to out[]int32
func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
out = buffer
inpos := cursor.New()
outpos := cursor.New()
if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
return nil, err
}
return out[:outpos.Get()], nil
}
func (this *FastPFOR) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
inlength = encoding.FloorBy(inlength, DefaultBlockSize)
if inlength == 0 {
return errors.New("fastpfor/Compress: inlength = 0. No work done.")
}
out[outpos.Get()] = int32(inlength)
outpos.Increment()
initoffset := cursor.New()
copy(this.dataPointers, zeroDataPointers)
copy(this.freqs, zeroFreqs)
finalInpos := inpos.Get() + inlength
for inpos.Get() != finalInpos {
thissize := int(math.Min(float64(this.pageSize), float64(finalInpos-inpos.Get())))
if err := this.encodePage(in, inpos, thissize, out, outpos, initoffset); err != nil {
return errors.New("fastpfor/Compress: " + err.Error())
}
}
return nil
}
func uncompress(arr []int32) ([]int32, error) {
newinpos := cursor.New()
newoutpos := cursor.New()
recov := make([]int32, 10000)
codec := bp32.New()
e := codec.Uncompress(arr, newinpos, len(arr)-1, recov, newoutpos)
l := 0
for i, v := range recov[:newoutpos.Get()] {
if v != 0 {
l = i
}
}
return recov[:l+1], e
}
// go test -bench=Decode
func BenchmarkDecode(b *testing.B) {
b.StopTimer()
length := 128 * 1024
data := generators.GenerateClustered(length, 1<<24)
compdata := make([]int32, 2*length)
recov := make([]int32, length)
inpos := cursor.New()
outpos := cursor.New()
codec := New()
codec.Compress(data, inpos, len(data), compdata, outpos)
b.StartTimer()
for j := 0; j < b.N; j++ {
newinpos := cursor.New()
newoutpos := cursor.New()
codec.Uncompress(compdata, newinpos, outpos.Get()-newinpos.Get(), recov, newoutpos)
}
}
func compress(in []int32) (arr []int32, e error) {
l := len(in)
if len(in)%128 != 0 {
l = l + (128 - len(in)%128)
}
arr = make([]int32, l)
copy(arr, in)
compdata := make([]int32, 2*l)
inpos := cursor.New()
outpos := cursor.New()
codec := bp32.New()
e = codec.Compress(arr, inpos, l, compdata, outpos)
if e != nil {
return nil, e
}
return compdata[:outpos.Get()+1], e
}
func (this *FastPFOR) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
if inlength == 0 {
return errors.New("fastpfor/Uncompress: inlength = 0. No work done.")
}
mynvalue := in[inpos.Get()]
inpos.Increment()
initoffset := cursor.New()
copy(this.dataPointers, zeroDataPointers)
finalout := outpos.Get() + int(mynvalue)
for outpos.Get() != finalout {
thissize := int(math.Min(float64(this.pageSize), float64(finalout-outpos.Get())))
if err := this.decodePage(in, inpos, out, outpos, thissize, initoffset); err != nil {
return errors.New("fastpfor/Uncompress: " + err.Error())
}
}
return nil
}