func (this *DecodingTask) decode() {
buffer := this.buf
res := Message{blockId: this.currentBlockId}
// Wait for task processing the previous block to complete
if this.input != nil {
run := <-this.input
// If one of the previous tasks failed, skip
if run == false {
notify(this.output, this.result, false, res)
return
}
}
defer func() {
if r := recover(); r != nil {
// Error => cancel concurrent decoding tasks
res.err = NewIOError(r.(error).Error(), ERR_READ_FILE)
notify(this.output, this.result, false, res)
}
}()
// Extract header directly from bitstream
read := this.ibs.Read()
mode := byte(this.ibs.ReadBits(8))
var preTransformLength uint
checksum1 := uint32(0)
if (mode & SMALL_BLOCK_MASK) != 0 {
preTransformLength = uint(mode & COPY_LENGTH_MASK)
} else {
dataSize := uint(1 + (mode & 0x03))
length := dataSize << 3
mask := uint64(1<<length) - 1
preTransformLength = uint(this.ibs.ReadBits(length) & mask)
}
if preTransformLength == 0 {
// Last block is empty, return success and cancel pending tasks
res.decoded = 0
notify(this.output, this.result, false, res)
return
}
if preTransformLength > MAX_BITSTREAM_BLOCK_SIZE {
// Error => cancel concurrent decoding tasks
errMsg := fmt.Sprintf("Invalid compressed block length: %d", preTransformLength)
res.err = NewIOError(errMsg, ERR_BLOCK_SIZE)
notify(this.output, this.result, false, res)
return
}
// Extract checksum from bit stream (if any)
if this.hasher != nil {
checksum1 = uint32(this.ibs.ReadBits(32))
}
if len(this.listeners) > 0 {
// Notify before entropy (block size in bitstream is unknown)
evt := &BlockEvent{eventType: EVT_BEFORE_ENTROPY, blockId: this.currentBlockId,
blockSize: -1, hash: checksum1, time_: time.Now(),
hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
res.checksum = checksum1
if this.typeOfTransform == function.NULL_TRANSFORM_TYPE {
buffer = this.data // share buffers if no transform
} else {
bufferSize := this.blockLength
if bufferSize < preTransformLength {
bufferSize = preTransformLength
}
if len(buffer) < int(bufferSize) {
buffer = make([]byte, bufferSize)
}
}
// Each block is decoded separately
// Rebuild the entropy decoder to reset block statistics
ed, err := entropy.NewEntropyDecoder(this.ibs, this.typeOfEntropy)
if err != nil {
// Error => cancel concurrent decoding tasks
res.err = NewIOError(err.Error(), ERR_INVALID_CODEC)
notify(this.output, this.result, false, res)
return
}
defer ed.Dispose()
// Block entropy decode
if _, err = ed.Decode(buffer[0:preTransformLength]); err != nil {
// Error => cancel concurrent decoding tasks
res.err = NewIOError(err.Error(), ERR_PROCESS_BLOCK)
notify(this.output, this.result, false, res)
return
}
if len(this.listeners) > 0 {
// Notify after entropy
evt := &BlockEvent{eventType: EVT_AFTER_ENTROPY, blockId: this.currentBlockId,
blockSize: int((this.ibs.Read() - read) / 8), hash: checksum1,
time_: time.Now(), hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
// After completion of the entropy decoding, unfreeze the task processing
// the next block (if any)
notify(this.output, nil, true, res)
if len(this.listeners) > 0 {
// Notify before transform
evt := &BlockEvent{eventType: EVT_BEFORE_TRANSFORM, blockId: this.currentBlockId,
blockSize: int(preTransformLength), hash: checksum1, time_: time.Now(),
hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
read = this.ibs.Read() - read
if ((mode & SMALL_BLOCK_MASK) != 0) || ((mode & SKIP_FUNCTION_MASK) != 0) {
if !bytes.Equal(buffer, this.data) {
copy(this.data, buffer[0:preTransformLength])
}
res.decoded = int(preTransformLength)
} else {
transform, err := function.NewByteFunction(preTransformLength, this.typeOfTransform)
if err != nil {
// Error => return
res.err = NewIOError(err.Error(), ERR_INVALID_CODEC)
notify(nil, this.result, false, res)
return
}
var oIdx uint
// Inverse transform
if _, oIdx, err = transform.Inverse(buffer, this.data); err != nil {
// Error => return
res.err = NewIOError(err.Error(), ERR_PROCESS_BLOCK)
notify(nil, this.result, false, res)
return
}
res.decoded = int(oIdx)
// Verify checksum
if this.hasher != nil {
checksum2 := this.hasher.Hash(this.data[0:res.decoded])
if checksum2 != checksum1 {
errMsg := fmt.Sprintf("Corrupted bitstream: expected checksum %x, found %x", checksum1, checksum2)
res.err = NewIOError(errMsg, ERR_PROCESS_BLOCK)
notify(nil, this.result, false, res)
return
}
}
}
notify(nil, this.result, false, res)
}
func (this *EncodingTask) encode() {
transform, err := function.NewByteFunction(this.blockLength, this.typeOfTransform)
if err != nil {
<-this.input
this.output <- NewIOError(err.Error(), ERR_CREATE_CODEC)
return
}
buffer := this.buf
requiredSize := transform.MaxEncodedLen(int(this.blockLength))
if requiredSize == -1 {
// Max size unknown => guess
requiredSize = int(this.blockLength*5) >> 2
}
if this.typeOfTransform == function.NULL_TRANSFORM_TYPE {
buffer = this.data // share buffers if no transform
} else if len(buffer) < requiredSize {
buffer = make([]byte, requiredSize)
}
mode := byte(0)
dataSize := uint(0)
postTransformLength := this.blockLength
checksum := uint32(0)
iIdx := uint(0)
oIdx := uint(0)
// Compute block checksum
if this.hasher != nil {
checksum = this.hasher.Hash(this.data[0:this.blockLength])
}
if len(this.listeners) > 0 {
// Notify before transform
evt := &BlockEvent{eventType: EVT_BEFORE_TRANSFORM, blockId: this.currentBlockId,
blockSize: int(this.blockLength), hash: checksum, time_: time.Now(),
hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
if this.blockLength <= SMALL_BLOCK_SIZE {
// Just copy
if !kanzi.SameByteSlices(buffer, this.data, false) {
copy(buffer, this.data[0:this.blockLength])
}
iIdx += this.blockLength
oIdx += this.blockLength
mode = byte(SMALL_BLOCK_SIZE | (this.blockLength & COPY_LENGTH_MASK))
} else {
// Forward transform
iIdx, oIdx, err = transform.Forward(this.data, buffer)
if err != nil {
// Transform failed (probably due to lack of space in output buffer)
if !kanzi.SameByteSlices(buffer, this.data, false) {
copy(buffer, this.data)
}
iIdx = this.blockLength
oIdx = this.blockLength
mode |= SKIP_FUNCTION_MASK
}
postTransformLength = oIdx
for i := uint64(0xFF); i < uint64(postTransformLength); i <<= 8 {
dataSize++
}
if dataSize > 3 {
<-this.input
this.output <- NewIOError("Invalid block data length", ERR_WRITE_FILE)
return
}
// Record size of 'block size' - 1 in bytes
mode |= byte(dataSize & 0x03)
dataSize++
}
if len(this.listeners) > 0 {
// Notify after transform
evt := &BlockEvent{eventType: EVT_AFTER_TRANSFORM, blockId: this.currentBlockId,
blockSize: int(postTransformLength), hash: checksum, time_: time.Now(),
hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
// Wait for the concurrent task processing the previous block to complete
// entropy encoding. Entropy encoding must happen sequentially (and
// in the correct block order) in the bitstream.
err2 := <-this.input
if err2 != nil {
this.output <- err2
return
}
// Each block is encoded separately
// Rebuild the entropy encoder to reset block statistics
ee, err := entropy.NewEntropyEncoder(this.obs, this.typeOfEntropy)
if err != nil {
this.output <- NewIOError(err.Error(), ERR_CREATE_CODEC)
return
}
// Write block 'header' (mode + compressed length)
written := this.obs.Written()
this.obs.WriteBits(uint64(mode), 8)
if dataSize > 0 {
this.obs.WriteBits(uint64(postTransformLength), 8*dataSize)
}
// Write checksum
if this.hasher != nil {
this.obs.WriteBits(uint64(checksum), 32)
}
if len(this.listeners) > 0 {
// Notify before entropy
evt := &BlockEvent{eventType: EVT_BEFORE_ENTROPY, blockId: this.currentBlockId,
blockSize: int(postTransformLength), time_: time.Now(),
hash: checksum, hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
// Entropy encode block
_, err = ee.Encode(buffer[0:postTransformLength])
if err != nil {
this.output <- NewIOError(err.Error(), ERR_PROCESS_BLOCK)
return
}
// Dispose before displaying statistics. Dispose may write to the bitstream
ee.Dispose()
if len(this.listeners) > 0 {
// Notify after entropy
evt := &BlockEvent{eventType: EVT_AFTER_ENTROPY, blockId: this.currentBlockId,
blockSize: int((this.obs.Written() - written) / 8), time_: time.Now(),
hash: checksum, hashing: this.hasher != nil}
notifyListeners(this.listeners, evt)
}
// Notify of completion of the task
this.output <- error(nil)
}