Ejemplo n.º 1
0
func TestCorrectness() {
	fmt.Printf("\n\nCorrectness test")

	// Test behavior
	for ii := 1; ii < 20; ii++ {
		fmt.Printf("\nTest %v", ii)
		var values []byte
		rand.Seed(time.Now().UTC().UnixNano())

		if ii == 3 {
			values = []byte{0, 0, 32, 15, -4 & 0xFF, 16, 0, 16, 0, 7, -1 & 0xFF, -4 & 0xFF, -32 & 0xFF, 0, 31, -1 & 0xFF}
		} else if ii == 2 {
			values = []byte{0x3d, 0x4d, 0x54, 0x47, 0x5a, 0x36, 0x39, 0x26, 0x72, 0x6f, 0x6c, 0x65, 0x3d, 0x70, 0x72, 0x65}
		} else if ii == 4 {
			values = []byte{65, 71, 74, 66, 76, 65, 69, 77, 74, 79, 68, 75, 73, 72, 77, 68, 78, 65, 79, 79, 78, 66, 77, 71, 64, 70, 74, 77, 64, 67, 71, 64}
		} else if ii == 1 {
			values = make([]byte, 32)

			for i := range values {
				values[i] = byte(2) // all identical
			}
		} else if ii == 5 {
			values = make([]byte, 32)

			for i := range values {
				values[i] = byte(2 + (i & 1)) // 2 symbols
			}
		} else {
			values = make([]byte, 32)

			for i := range values {
				values[i] = byte(64 + 3*ii + rand.Intn(ii+1))
			}
		}

		fmt.Printf("\nOriginal: ")

		for i := range values {
			fmt.Printf("%d ", values[i])
		}

		fmt.Printf("\nEncoded: ")
		buffer := make([]byte, 16384)
		oFile, _ := util.NewByteArrayOutputStream(buffer, true)
		defer oFile.Close()
		obs, _ := bitstream.NewDefaultOutputBitStream(oFile, 16384)
		dbgbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)
		dbgbs.ShowByte(true)
		dbgbs.Mark(true)
		hc, _ := entropy.NewANSRangeEncoder(dbgbs)

		if _, err := hc.Encode(values); err != nil {
			fmt.Printf("Error during encoding: %s", err)
			os.Exit(1)
		}

		hc.Dispose()
		dbgbs.Close()
		println()

		iFile, _ := util.NewByteArrayInputStream(buffer, true)
		defer iFile.Close()
		ibs, _ := bitstream.NewDefaultInputBitStream(iFile, 16384)
		dbgbs2, _ := bitstream.NewDebugInputBitStream(ibs, os.Stdout)
		//dbgbs2.ShowByte(true)
		dbgbs2.Mark(true)

		hd, _ := entropy.NewANSRangeDecoder(dbgbs2)

		ok := true
		values2 := make([]byte, len(values))
		if _, err := hd.Decode(values2); err != nil {
			fmt.Printf("Error during decoding: %s", err)
			os.Exit(1)
		}

		fmt.Printf("\nDecoded: ")

		for i := range values2 {
			fmt.Printf("%v ", values2[i])

			if values[i] != values2[i] {
				ok = false
			}
		}

		if ok == true {
			fmt.Printf("\nIdentical")
		} else {
			fmt.Printf("\n! *** Different *** !")
			os.Exit(1)
		}

		hd.Dispose()
		println()
	}
}
Ejemplo n.º 2
0
func TestSpeed() {
	fmt.Printf("\n\nSpeed test\n")
	repeats := []int{3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3}

	for jj := 0; jj < 3; jj++ {
		fmt.Printf("Test %v\n", jj+1)
		delta1 := int64(0)
		delta2 := int64(0)
		iter := 4000
		size := 50000
		buffer := make([]byte, size*2)
		values1 := make([]byte, size)
		values2 := make([]byte, size)

		for ii := 0; ii < iter; ii++ {
			idx := jj

			for i := 0; i < len(values1); i++ {
				i0 := i

				length := repeats[idx]
				idx = (idx + 1) & 0x0F

				if i0+length >= len(values1) {
					length = 1
				}

				for j := i0; j < i0+length; j++ {
					values1[j] = byte(uint(i0) & 0xFF)
					i++
				}
			}

			oFile, _ := util.NewByteArrayOutputStream(buffer, false)
			defer oFile.Close()
			obs, _ := bitstream.NewDefaultOutputBitStream(oFile, uint(size))
			rc, _ := entropy.NewANSRangeEncoder(obs)

			// Encode
			before := time.Now()

			if _, err := rc.Encode(values1); err != nil {
				fmt.Printf("An error occured during encoding: %v\n", err)
				os.Exit(1)
			}

			rc.Dispose()

			if _, err := obs.Close(); err != nil {
				fmt.Printf("Error during close: %v\n", err)
				os.Exit(1)
			}

			after := time.Now()
			delta1 += after.Sub(before).Nanoseconds()
		}

		for ii := 0; ii < iter; ii++ {
			iFile, _ := util.NewByteArrayInputStream(buffer, false)
			defer iFile.Close()
			ibs, _ := bitstream.NewDefaultInputBitStream(iFile, uint(size))
			rd, _ := entropy.NewANSRangeDecoder(ibs)

			// Decode
			before := time.Now()

			if _, err := rd.Decode(values2); err != nil {
				fmt.Printf("An error occured during decoding: %v\n", err)
				os.Exit(1)
			}

			rd.Dispose()

			if _, err := ibs.Close(); err != nil {
				fmt.Printf("Error during close: %v\n", err)
				os.Exit(1)
			}

			after := time.Now()
			delta2 += after.Sub(before).Nanoseconds()
		}

		fmt.Printf("Encode [ms]      : %d\n", delta1/1000000)
		fmt.Printf("Throughput [KB/s]: %d\n", (int64(iter*size))*1000000/delta1*1000/1024)
		fmt.Printf("Decode [ms]      : %d\n", delta2/1000000)
		fmt.Printf("Throughput [KB/s]: %d\n", (int64(iter*size))*1000000/delta2*1000/1024)
	}
}
Ejemplo n.º 3
0
func testCorrectnessAligned() {
	fmt.Printf("Correctness Test - byte aligned\n")
	values := make([]int, 100)
	rand.Seed(time.Now().UTC().UnixNano())

	for test := 0; test < 10; test++ {
		buffer := make([]byte, 16384)
		os_, _ := util.NewByteArrayOutputStream(buffer, true)
		obs, _ := bitstream.NewDefaultOutputBitStream(os_, 16384)
		dbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)
		dbs.ShowByte(true)

		for i := range values {
			if test < 5 {
				values[i] = rand.Intn(test*1000 + 100)
			} else {
				values[i] = rand.Intn(1 << 31)
			}

			fmt.Printf("%v ", values[i])

			if i%50 == 49 {
				println()
			}
		}

		println()
		println()

		for i := range values {
			dbs.WriteBits(uint64(values[i]), 32)
		}

		// Close first to force flush()
		dbs.Close()

		is_, _ := util.NewByteArrayInputStream(buffer, true)
		ibs, _ := bitstream.NewDefaultInputBitStream(is_, 16384)
		fmt.Printf("\nRead:\n")
		ok := true

		for i := range values {
			x := ibs.ReadBits(32)
			fmt.Printf("%v", x)

			if int(x) == values[i] {
				fmt.Printf(" ")
			} else {
				fmt.Printf("* ")
				ok = false
			}

			if i%50 == 49 {
				println()
			}
		}

		ibs.Close()
		println()
		println()
		fmt.Printf("Bits written: %v\n", dbs.Written())
		fmt.Printf("Bits read: %v\n", ibs.Read())

		if ok {
			fmt.Printf("\nSuccess\n")
		} else {
			fmt.Printf("\nFailure\n")
		}

		println()
		println()
	}

}
Ejemplo n.º 4
0
func testSpeed() {
	fmt.Printf("Speed Test\n")
	var filename = flag.String("filename", "r:\\output.bin", "Ouput file name for speed test")
	flag.Parse()

	values := []uint64{3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3,
		31, 14, 41, 15, 59, 92, 26, 65, 53, 35, 58, 89, 97, 79, 93, 32}
	iter := 150
	read := uint64(0)
	written := uint64(0)
	delta1 := int64(0)
	delta2 := int64(0)
	nn := 100000 * len(values)
	defer os.Remove(*filename)

	for test := 0; test < iter; test++ {
		file1, err := os.Create(*filename)

		if err != nil {
			fmt.Printf("Cannot create %s", *filename)

			return
		}

		bos, _ := io.NewBufferedOutputStream(file1)
		obs, _ := bitstream.NewDefaultOutputBitStream(bos, 16*1024)
		before := time.Now()
		for i := 0; i < nn; i++ {
			obs.WriteBits(values[i%len(values)], 1+uint(i&63))
		}

		// Close first to force flush()
		obs.Close()
		delta1 += time.Now().Sub(before).Nanoseconds()
		written += obs.Written()
		file1.Close()

		file2, err := os.Open(*filename)

		if err != nil {
			fmt.Printf("Cannot open %s", *filename)

			return
		}

		bis, _ := io.NewBufferedInputStream(file2)
		ibs, _ := bitstream.NewDefaultInputBitStream(bis, 1024*1024)
		before = time.Now()

		for i := 0; i < nn; i++ {
			ibs.ReadBits(1 + uint(i&63))
		}

		ibs.Close()
		delta2 += time.Now().Sub(before).Nanoseconds()
		read += ibs.Read()
		file2.Close()
	}

	println()
	fmt.Printf("%v bits written (%v MB)\n", written, written/1024/8192)
	fmt.Printf("%v bits read (%v MB)\n", read, read/1024/8192)
	println()
	fmt.Printf("Write [ms]        : %v\n", delta1/1000000)
	fmt.Printf("Throughput [MB/s] : %d\n", (written/1024*1000/8192)/uint64(delta1/1000000))
	fmt.Printf("Read [ms]         : %v\n", delta2/1000000)
	fmt.Printf("Throughput [MB/s] : %d\n", (read/1024*1000/8192)/uint64(delta2/1000000))
}
Ejemplo n.º 5
0
func TestCorrectness() {
	fmt.Printf("\n\nCorrectness test")

	// Test behavior
	for ii := 1; ii < 20; ii++ {
		fmt.Printf("\nTest %v", ii)
		var values []byte
		rand.Seed(time.Now().UTC().UnixNano())

		if ii == 1 {
			values = []byte{17, 8, 30, 28, 6, 26, 2, 9, 31, 0, 15, 30, 11, 27, 17, 11, 24, 6, 10, 24, 15, 10, 16, 13, 6, 21, 1, 18, 0, 3, 23, 6}
		} else {
			values = make([]byte, 32)

			for i := range values {
				values[i] = byte(rand.Intn(32) - 16*(ii&1))
			}
		}

		fmt.Printf("\nOriginal: ")

		for i := range values {
			fmt.Printf("%d ", values[i])
		}

		signed := true

		if ii&1 == 0 {
			signed = false
		}

		fmt.Printf("\nEncoded: ")
		buffer := make([]byte, 16384)
		oFile, _ := util.NewByteArrayOutputStream(buffer, true)
		defer oFile.Close()
		obs, _ := bitstream.NewDefaultOutputBitStream(oFile, 16384)
		dbgbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)

		// Alternate signed / unsigned coding
		fpc, _ := entropy.NewExpGolombEncoder(dbgbs, signed)

		if _, err := fpc.Encode(values); err != nil {
			fmt.Printf("Error during encoding: %s", err)
			os.Exit(1)
		}

		fpc.Dispose()
		dbgbs.Close()

		if _, err := dbgbs.Close(); err != nil {
			fmt.Printf("Error during close: %v\n", err)
			os.Exit(1)
		}

		println()

		iFile, _ := util.NewByteArrayInputStream(buffer, true)
		defer iFile.Close()
		ibs, _ := bitstream.NewDefaultInputBitStream(iFile, 16384)
		dbgbs2, _ := bitstream.NewDebugInputBitStream(ibs, os.Stdout)
		dbgbs2.Mark(true)

		fpd, _ := entropy.NewExpGolombDecoder(dbgbs2, signed)

		ok := true
		values2 := make([]byte, len(values))

		if _, err := fpd.Decode(values2); err != nil {
			fmt.Printf("Error during decoding: %s", err)
			os.Exit(1)
		}

		fmt.Printf("\nDecoded: ")

		for i := range values2 {
			fmt.Printf("%v ", values2[i])

			if values[i] != values2[i] {
				ok = false
			}
		}

		if ok == true {
			fmt.Printf("\nIdentical")
		} else {
			fmt.Printf("\n! *** Different *** !")
			os.Exit(1)
		}

		fpd.Dispose()
		dbgbs2.Close()
		fmt.Printf("\n")
	}
}
Ejemplo n.º 6
0
func NewCompressedOutputStream(entropyCodec string, functionType string, os io.WriteCloser, blockSize uint,
	checksum bool, debugWriter io.Writer, jobs uint) (*CompressedOutputStream, error) {
	if os == nil {
		return nil, NewIOError("Invalid null output stream parameter", ERR_CREATE_STREAM)
	}

	if blockSize > MAX_BITSTREAM_BLOCK_SIZE {
		errMsg := fmt.Sprintf("The block size must be at most %d MB", MAX_BITSTREAM_BLOCK_SIZE>>20)
		return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
	}

	if blockSize < MIN_BITSTREAM_BLOCK_SIZE {
		errMsg := fmt.Sprintf("The block size must be at least %d", MIN_BITSTREAM_BLOCK_SIZE)
		return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
	}

	if int(blockSize)&-16 != int(blockSize) {
		return nil, NewIOError("The block size must be a multiple of 16", ERR_CREATE_STREAM)
	}

	if jobs < 1 || jobs > 16 {
		return nil, NewIOError("The number of jobs must be in [1..16]", ERR_CREATE_STREAM)
	}

	this := new(CompressedOutputStream)
	var err error

	bufferSize := blockSize

	if bufferSize > 65536 {
		bufferSize = 65536
	}

	if this.obs, err = bitstream.NewDefaultOutputBitStream(os, bufferSize); err != nil {
		return nil, err
	}

	// Check entropy type validity (panic on error)
	this.entropyType = entropy.GetEntropyCodecType(entropyCodec)

	// Check transform type validity (panic on error)
	this.transformType = function.GetByteFunctionType(functionType)

	this.blockSize = blockSize

	if checksum == true {
		this.hasher, err = util.NewXXHash(BITSTREAM_TYPE)

		if err != nil {
			return nil, err
		}
	}

	this.data = make([]byte, jobs*blockSize)
	this.buffers = make([][]byte, jobs)

	for i := range this.buffers {
		this.buffers[i] = EMPTY_BYTE_SLICE
	}

	this.debugWriter = debugWriter
	this.jobs = int(jobs)
	this.blockId = 0
	this.channels = make([]chan error, this.jobs+1)

	for i := range this.channels {
		this.channels[i] = make(chan error)
	}

	this.listeners = make([]BlockListener, 0)
	return this, nil
}