Exemplo n.º 1
0
func RunCompress(codec encoding.Integer, in []int32, length int, prof bool) (duration int64, out []int32, err error) {
	out = make([]int32, length*2)
	inpos := cursor.New()
	outpos := cursor.New()

	now := time.Now()
	if prof {
		f, e := os.Create("cpu.compress.pprof")
		if e != nil {
			log.Fatal(e)
		}
		defer f.Close()

		pprof.StartCPUProfile(f)
	}

	if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
		return 0, nil, err
	}
	since := time.Since(now).Nanoseconds()

	if prof {
		pprof.StopCPUProfile()
	}

	return since, out[:outpos.Get()], nil
}
Exemplo n.º 2
0
// Compress compresses in[]int32 to out[]int32
func Compress32(in []int32) (out []int32, err error) {
	out = make([]int32, len(in)*2)
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}
Exemplo n.º 3
0
// Uncompress uncompresses in[]int32 to out[]int32
func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
	out = buffer
	inpos := cursor.New()
	outpos := cursor.New()

	if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
		return nil, err
	}

	return out[:outpos.Get()], nil
}
Exemplo n.º 4
0
func (this *FastPFOR) Compress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
	inlength = encoding.FloorBy(inlength, DefaultBlockSize)

	if inlength == 0 {
		return errors.New("fastpfor/Compress: inlength = 0. No work done.")
	}

	out[outpos.Get()] = int32(inlength)
	outpos.Increment()

	initoffset := cursor.New()

	copy(this.dataPointers, zeroDataPointers)
	copy(this.freqs, zeroFreqs)

	finalInpos := inpos.Get() + inlength

	for inpos.Get() != finalInpos {
		thissize := int(math.Min(float64(this.pageSize), float64(finalInpos-inpos.Get())))

		if err := this.encodePage(in, inpos, thissize, out, outpos, initoffset); err != nil {
			return errors.New("fastpfor/Compress: " + err.Error())
		}
	}

	return nil
}
Exemplo n.º 5
0
func uncompress(arr []int32) ([]int32, error) {
	newinpos := cursor.New()
	newoutpos := cursor.New()

	recov := make([]int32, 10000)

	codec := bp32.New()
	e := codec.Uncompress(arr, newinpos, len(arr)-1, recov, newoutpos)
	l := 0
	for i, v := range recov[:newoutpos.Get()] {
		if v != 0 {
			l = i
		}
	}
	return recov[:l+1], e
}
Exemplo n.º 6
0
// go test -bench=Decode
func BenchmarkDecode(b *testing.B) {
	b.StopTimer()
	length := 128 * 1024
	data := generators.GenerateClustered(length, 1<<24)
	compdata := make([]int32, 2*length)
	recov := make([]int32, length)
	inpos := cursor.New()
	outpos := cursor.New()
	codec := New()
	codec.Compress(data, inpos, len(data), compdata, outpos)
	b.StartTimer()
	for j := 0; j < b.N; j++ {
		newinpos := cursor.New()
		newoutpos := cursor.New()
		codec.Uncompress(compdata, newinpos, outpos.Get()-newinpos.Get(), recov, newoutpos)
	}
}
Exemplo n.º 7
0
func compress(in []int32) (arr []int32, e error) {
	l := len(in)
	if len(in)%128 != 0 {
		l = l + (128 - len(in)%128)
	}

	arr = make([]int32, l)
	copy(arr, in)

	compdata := make([]int32, 2*l)
	inpos := cursor.New()
	outpos := cursor.New()
	codec := bp32.New()
	e = codec.Compress(arr, inpos, l, compdata, outpos)
	if e != nil {
		return nil, e
	}

	return compdata[:outpos.Get()+1], e
}
Exemplo n.º 8
0
func (this *FastPFOR) Uncompress(in []int32, inpos *cursor.Cursor, inlength int, out []int32, outpos *cursor.Cursor) error {
	if inlength == 0 {
		return errors.New("fastpfor/Uncompress: inlength = 0. No work done.")
	}

	mynvalue := in[inpos.Get()]
	inpos.Increment()

	initoffset := cursor.New()

	copy(this.dataPointers, zeroDataPointers)

	finalout := outpos.Get() + int(mynvalue)
	for outpos.Get() != finalout {
		thissize := int(math.Min(float64(this.pageSize), float64(finalout-outpos.Get())))

		if err := this.decodePage(in, inpos, out, outpos, thissize, initoffset); err != nil {
			return errors.New("fastpfor/Uncompress: " + err.Error())
		}
	}
	return nil
}