func TestCorrectness() {
fmt.Printf("\n\nCorrectness test")
// Test behavior
for ii := 1; ii < 20; ii++ {
fmt.Printf("\nTest %v", ii)
var values []byte
rand.Seed(time.Now().UTC().UnixNano())
if ii == 3 {
values = []byte{0, 0, 32, 15, -4 & 0xFF, 16, 0, 16, 0, 7, -1 & 0xFF, -4 & 0xFF, -32 & 0xFF, 0, 31, -1 & 0xFF}
} else if ii == 2 {
values = []byte{0x3d, 0x4d, 0x54, 0x47, 0x5a, 0x36, 0x39, 0x26, 0x72, 0x6f, 0x6c, 0x65, 0x3d, 0x70, 0x72, 0x65}
} else if ii == 4 {
values = []byte{65, 71, 74, 66, 76, 65, 69, 77, 74, 79, 68, 75, 73, 72, 77, 68, 78, 65, 79, 79, 78, 66, 77, 71, 64, 70, 74, 77, 64, 67, 71, 64}
} else if ii == 1 {
values = make([]byte, 32)
for i := range values {
values[i] = byte(2) // all identical
}
} else if ii == 5 {
values = make([]byte, 32)
for i := range values {
values[i] = byte(2 + (i & 1)) // 2 symbols
}
} else {
values = make([]byte, 32)
for i := range values {
values[i] = byte(64 + 3*ii + rand.Intn(ii+1))
}
}
fmt.Printf("\nOriginal: ")
for i := range values {
fmt.Printf("%d ", values[i])
}
fmt.Printf("\nEncoded: ")
buffer := make([]byte, 16384)
oFile, _ := util.NewByteArrayOutputStream(buffer, true)
defer oFile.Close()
obs, _ := bitstream.NewDefaultOutputBitStream(oFile, 16384)
dbgbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)
dbgbs.ShowByte(true)
dbgbs.Mark(true)
hc, _ := entropy.NewANSRangeEncoder(dbgbs)
if _, err := hc.Encode(values); err != nil {
fmt.Printf("Error during encoding: %s", err)
os.Exit(1)
}
hc.Dispose()
dbgbs.Close()
println()
iFile, _ := util.NewByteArrayInputStream(buffer, true)
defer iFile.Close()
ibs, _ := bitstream.NewDefaultInputBitStream(iFile, 16384)
dbgbs2, _ := bitstream.NewDebugInputBitStream(ibs, os.Stdout)
//dbgbs2.ShowByte(true)
dbgbs2.Mark(true)
hd, _ := entropy.NewANSRangeDecoder(dbgbs2)
ok := true
values2 := make([]byte, len(values))
if _, err := hd.Decode(values2); err != nil {
fmt.Printf("Error during decoding: %s", err)
os.Exit(1)
}
fmt.Printf("\nDecoded: ")
for i := range values2 {
fmt.Printf("%v ", values2[i])
if values[i] != values2[i] {
ok = false
}
}
if ok == true {
fmt.Printf("\nIdentical")
} else {
fmt.Printf("\n! *** Different *** !")
os.Exit(1)
}
hd.Dispose()
println()
}
}
func TestSpeed() {
fmt.Printf("\n\nSpeed test\n")
repeats := []int{3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3}
for jj := 0; jj < 3; jj++ {
fmt.Printf("Test %v\n", jj+1)
delta1 := int64(0)
delta2 := int64(0)
iter := 4000
size := 50000
buffer := make([]byte, size*2)
values1 := make([]byte, size)
values2 := make([]byte, size)
for ii := 0; ii < iter; ii++ {
idx := jj
for i := 0; i < len(values1); i++ {
i0 := i
length := repeats[idx]
idx = (idx + 1) & 0x0F
if i0+length >= len(values1) {
length = 1
}
for j := i0; j < i0+length; j++ {
values1[j] = byte(uint(i0) & 0xFF)
i++
}
}
oFile, _ := util.NewByteArrayOutputStream(buffer, false)
defer oFile.Close()
obs, _ := bitstream.NewDefaultOutputBitStream(oFile, uint(size))
rc, _ := entropy.NewANSRangeEncoder(obs)
// Encode
before := time.Now()
if _, err := rc.Encode(values1); err != nil {
fmt.Printf("An error occured during encoding: %v\n", err)
os.Exit(1)
}
rc.Dispose()
if _, err := obs.Close(); err != nil {
fmt.Printf("Error during close: %v\n", err)
os.Exit(1)
}
after := time.Now()
delta1 += after.Sub(before).Nanoseconds()
}
for ii := 0; ii < iter; ii++ {
iFile, _ := util.NewByteArrayInputStream(buffer, false)
defer iFile.Close()
ibs, _ := bitstream.NewDefaultInputBitStream(iFile, uint(size))
rd, _ := entropy.NewANSRangeDecoder(ibs)
// Decode
before := time.Now()
if _, err := rd.Decode(values2); err != nil {
fmt.Printf("An error occured during decoding: %v\n", err)
os.Exit(1)
}
rd.Dispose()
if _, err := ibs.Close(); err != nil {
fmt.Printf("Error during close: %v\n", err)
os.Exit(1)
}
after := time.Now()
delta2 += after.Sub(before).Nanoseconds()
}
fmt.Printf("Encode [ms] : %d\n", delta1/1000000)
fmt.Printf("Throughput [KB/s]: %d\n", (int64(iter*size))*1000000/delta1*1000/1024)
fmt.Printf("Decode [ms] : %d\n", delta2/1000000)
fmt.Printf("Throughput [KB/s]: %d\n", (int64(iter*size))*1000000/delta2*1000/1024)
}
}
func testCorrectnessAligned() {
fmt.Printf("Correctness Test - byte aligned\n")
values := make([]int, 100)
rand.Seed(time.Now().UTC().UnixNano())
for test := 0; test < 10; test++ {
buffer := make([]byte, 16384)
os_, _ := util.NewByteArrayOutputStream(buffer, true)
obs, _ := bitstream.NewDefaultOutputBitStream(os_, 16384)
dbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)
dbs.ShowByte(true)
for i := range values {
if test < 5 {
values[i] = rand.Intn(test*1000 + 100)
} else {
values[i] = rand.Intn(1 << 31)
}
fmt.Printf("%v ", values[i])
if i%50 == 49 {
println()
}
}
println()
println()
for i := range values {
dbs.WriteBits(uint64(values[i]), 32)
}
// Close first to force flush()
dbs.Close()
is_, _ := util.NewByteArrayInputStream(buffer, true)
ibs, _ := bitstream.NewDefaultInputBitStream(is_, 16384)
fmt.Printf("\nRead:\n")
ok := true
for i := range values {
x := ibs.ReadBits(32)
fmt.Printf("%v", x)
if int(x) == values[i] {
fmt.Printf(" ")
} else {
fmt.Printf("* ")
ok = false
}
if i%50 == 49 {
println()
}
}
ibs.Close()
println()
println()
fmt.Printf("Bits written: %v\n", dbs.Written())
fmt.Printf("Bits read: %v\n", ibs.Read())
if ok {
fmt.Printf("\nSuccess\n")
} else {
fmt.Printf("\nFailure\n")
}
println()
println()
}
}
func testSpeed() {
fmt.Printf("Speed Test\n")
var filename = flag.String("filename", "r:\\output.bin", "Ouput file name for speed test")
flag.Parse()
values := []uint64{3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3,
31, 14, 41, 15, 59, 92, 26, 65, 53, 35, 58, 89, 97, 79, 93, 32}
iter := 150
read := uint64(0)
written := uint64(0)
delta1 := int64(0)
delta2 := int64(0)
nn := 100000 * len(values)
defer os.Remove(*filename)
for test := 0; test < iter; test++ {
file1, err := os.Create(*filename)
if err != nil {
fmt.Printf("Cannot create %s", *filename)
return
}
bos, _ := io.NewBufferedOutputStream(file1)
obs, _ := bitstream.NewDefaultOutputBitStream(bos, 16*1024)
before := time.Now()
for i := 0; i < nn; i++ {
obs.WriteBits(values[i%len(values)], 1+uint(i&63))
}
// Close first to force flush()
obs.Close()
delta1 += time.Now().Sub(before).Nanoseconds()
written += obs.Written()
file1.Close()
file2, err := os.Open(*filename)
if err != nil {
fmt.Printf("Cannot open %s", *filename)
return
}
bis, _ := io.NewBufferedInputStream(file2)
ibs, _ := bitstream.NewDefaultInputBitStream(bis, 1024*1024)
before = time.Now()
for i := 0; i < nn; i++ {
ibs.ReadBits(1 + uint(i&63))
}
ibs.Close()
delta2 += time.Now().Sub(before).Nanoseconds()
read += ibs.Read()
file2.Close()
}
println()
fmt.Printf("%v bits written (%v MB)\n", written, written/1024/8192)
fmt.Printf("%v bits read (%v MB)\n", read, read/1024/8192)
println()
fmt.Printf("Write [ms] : %v\n", delta1/1000000)
fmt.Printf("Throughput [MB/s] : %d\n", (written/1024*1000/8192)/uint64(delta1/1000000))
fmt.Printf("Read [ms] : %v\n", delta2/1000000)
fmt.Printf("Throughput [MB/s] : %d\n", (read/1024*1000/8192)/uint64(delta2/1000000))
}
func TestCorrectness() {
fmt.Printf("\n\nCorrectness test")
// Test behavior
for ii := 1; ii < 20; ii++ {
fmt.Printf("\nTest %v", ii)
var values []byte
rand.Seed(time.Now().UTC().UnixNano())
if ii == 1 {
values = []byte{17, 8, 30, 28, 6, 26, 2, 9, 31, 0, 15, 30, 11, 27, 17, 11, 24, 6, 10, 24, 15, 10, 16, 13, 6, 21, 1, 18, 0, 3, 23, 6}
} else {
values = make([]byte, 32)
for i := range values {
values[i] = byte(rand.Intn(32) - 16*(ii&1))
}
}
fmt.Printf("\nOriginal: ")
for i := range values {
fmt.Printf("%d ", values[i])
}
signed := true
if ii&1 == 0 {
signed = false
}
fmt.Printf("\nEncoded: ")
buffer := make([]byte, 16384)
oFile, _ := util.NewByteArrayOutputStream(buffer, true)
defer oFile.Close()
obs, _ := bitstream.NewDefaultOutputBitStream(oFile, 16384)
dbgbs, _ := bitstream.NewDebugOutputBitStream(obs, os.Stdout)
// Alternate signed / unsigned coding
fpc, _ := entropy.NewExpGolombEncoder(dbgbs, signed)
if _, err := fpc.Encode(values); err != nil {
fmt.Printf("Error during encoding: %s", err)
os.Exit(1)
}
fpc.Dispose()
dbgbs.Close()
if _, err := dbgbs.Close(); err != nil {
fmt.Printf("Error during close: %v\n", err)
os.Exit(1)
}
println()
iFile, _ := util.NewByteArrayInputStream(buffer, true)
defer iFile.Close()
ibs, _ := bitstream.NewDefaultInputBitStream(iFile, 16384)
dbgbs2, _ := bitstream.NewDebugInputBitStream(ibs, os.Stdout)
dbgbs2.Mark(true)
fpd, _ := entropy.NewExpGolombDecoder(dbgbs2, signed)
ok := true
values2 := make([]byte, len(values))
if _, err := fpd.Decode(values2); err != nil {
fmt.Printf("Error during decoding: %s", err)
os.Exit(1)
}
fmt.Printf("\nDecoded: ")
for i := range values2 {
fmt.Printf("%v ", values2[i])
if values[i] != values2[i] {
ok = false
}
}
if ok == true {
fmt.Printf("\nIdentical")
} else {
fmt.Printf("\n! *** Different *** !")
os.Exit(1)
}
fpd.Dispose()
dbgbs2.Close()
fmt.Printf("\n")
}
}
func NewCompressedOutputStream(entropyCodec string, functionType string, os io.WriteCloser, blockSize uint,
checksum bool, debugWriter io.Writer, jobs uint) (*CompressedOutputStream, error) {
if os == nil {
return nil, NewIOError("Invalid null output stream parameter", ERR_CREATE_STREAM)
}
if blockSize > MAX_BITSTREAM_BLOCK_SIZE {
errMsg := fmt.Sprintf("The block size must be at most %d MB", MAX_BITSTREAM_BLOCK_SIZE>>20)
return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
}
if blockSize < MIN_BITSTREAM_BLOCK_SIZE {
errMsg := fmt.Sprintf("The block size must be at least %d", MIN_BITSTREAM_BLOCK_SIZE)
return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
}
if int(blockSize)&-16 != int(blockSize) {
return nil, NewIOError("The block size must be a multiple of 16", ERR_CREATE_STREAM)
}
if jobs < 1 || jobs > 16 {
return nil, NewIOError("The number of jobs must be in [1..16]", ERR_CREATE_STREAM)
}
this := new(CompressedOutputStream)
var err error
bufferSize := blockSize
if bufferSize > 65536 {
bufferSize = 65536
}
if this.obs, err = bitstream.NewDefaultOutputBitStream(os, bufferSize); err != nil {
return nil, err
}
// Check entropy type validity (panic on error)
this.entropyType = entropy.GetEntropyCodecType(entropyCodec)
// Check transform type validity (panic on error)
this.transformType = function.GetByteFunctionType(functionType)
this.blockSize = blockSize
if checksum == true {
this.hasher, err = util.NewXXHash(BITSTREAM_TYPE)
if err != nil {
return nil, err
}
}
this.data = make([]byte, jobs*blockSize)
this.buffers = make([][]byte, jobs)
for i := range this.buffers {
this.buffers[i] = EMPTY_BYTE_SLICE
}
this.debugWriter = debugWriter
this.jobs = int(jobs)
this.blockId = 0
this.channels = make([]chan error, this.jobs+1)
for i := range this.channels {
this.channels[i] = make(chan error)
}
this.listeners = make([]BlockListener, 0)
return this, nil
}