Example #1
0
func (this *DecodingTask) decode() {
	buffer := this.buf
	res := Message{blockId: this.currentBlockId}

	// Wait for task processing the previous block to complete
	if this.input != nil {
		run := <-this.input

		// If one of the previous tasks failed, skip
		if run == false {
			notify(this.output, this.result, false, res)
			return
		}
	}

	defer func() {
		if r := recover(); r != nil {
			// Error => cancel concurrent decoding tasks
			res.err = NewIOError(r.(error).Error(), ERR_READ_FILE)
			notify(this.output, this.result, false, res)
		}
	}()

	// Extract header directly from bitstream
	read := this.ibs.Read()
	mode := byte(this.ibs.ReadBits(8))
	var preTransformLength uint
	checksum1 := uint32(0)

	if (mode & SMALL_BLOCK_MASK) != 0 {
		preTransformLength = uint(mode & COPY_LENGTH_MASK)
	} else {
		dataSize := uint(1 + (mode & 0x03))
		length := dataSize << 3
		mask := uint64(1<<length) - 1
		preTransformLength = uint(this.ibs.ReadBits(length) & mask)
	}

	if preTransformLength == 0 {
		// Last block is empty, return success and cancel pending tasks
		res.decoded = 0
		notify(this.output, this.result, false, res)
		return
	}

	if preTransformLength > MAX_BITSTREAM_BLOCK_SIZE {
		// Error => cancel concurrent decoding tasks
		errMsg := fmt.Sprintf("Invalid compressed block length: %d", preTransformLength)
		res.err = NewIOError(errMsg, ERR_BLOCK_SIZE)
		notify(this.output, this.result, false, res)
		return
	}

	// Extract checksum from bit stream (if any)
	if this.hasher != nil {
		checksum1 = uint32(this.ibs.ReadBits(32))
	}

	if len(this.listeners) > 0 {
		// Notify before entropy (block size in bitstream is unknown)
		evt := &BlockEvent{eventType: EVT_BEFORE_ENTROPY, blockId: this.currentBlockId,
			blockSize: -1, hash: checksum1, time_: time.Now(),
			hashing: this.hasher != nil}

		notifyListeners(this.listeners, evt)
	}

	res.checksum = checksum1

	if this.typeOfTransform == function.NULL_TRANSFORM_TYPE {
		buffer = this.data // share buffers if no transform
	} else {
		bufferSize := this.blockLength

		if bufferSize < preTransformLength {
			bufferSize = preTransformLength
		}

		if len(buffer) < int(bufferSize) {
			buffer = make([]byte, bufferSize)
		}
	}

	// Each block is decoded separately
	// Rebuild the entropy decoder to reset block statistics
	ed, err := entropy.NewEntropyDecoder(this.ibs, this.typeOfEntropy)

	if err != nil {
		// Error => cancel concurrent decoding tasks
		res.err = NewIOError(err.Error(), ERR_INVALID_CODEC)
		notify(this.output, this.result, false, res)
		return
	}

	defer ed.Dispose()

	// Block entropy decode
	if _, err = ed.Decode(buffer[0:preTransformLength]); err != nil {
		// Error => cancel concurrent decoding tasks
		res.err = NewIOError(err.Error(), ERR_PROCESS_BLOCK)
		notify(this.output, this.result, false, res)
		return
	}

	if len(this.listeners) > 0 {
		// Notify after entropy
		evt := &BlockEvent{eventType: EVT_AFTER_ENTROPY, blockId: this.currentBlockId,
			blockSize: int((this.ibs.Read() - read) / 8), hash: checksum1,
			time_: time.Now(), hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	// After completion of the entropy decoding, unfreeze the task processing
	// the next block (if any)
	notify(this.output, nil, true, res)

	if len(this.listeners) > 0 {
		// Notify before transform
		evt := &BlockEvent{eventType: EVT_BEFORE_TRANSFORM, blockId: this.currentBlockId,
			blockSize: int(preTransformLength), hash: checksum1, time_: time.Now(),
			hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	read = this.ibs.Read() - read

	if ((mode & SMALL_BLOCK_MASK) != 0) || ((mode & SKIP_FUNCTION_MASK) != 0) {
		if !bytes.Equal(buffer, this.data) {
			copy(this.data, buffer[0:preTransformLength])
		}

		res.decoded = int(preTransformLength)
	} else {
		transform, err := function.NewByteFunction(preTransformLength, this.typeOfTransform)

		if err != nil {
			// Error => return
			res.err = NewIOError(err.Error(), ERR_INVALID_CODEC)
			notify(nil, this.result, false, res)
			return
		}

		var oIdx uint

		// Inverse transform
		if _, oIdx, err = transform.Inverse(buffer, this.data); err != nil {
			// Error => return
			res.err = NewIOError(err.Error(), ERR_PROCESS_BLOCK)
			notify(nil, this.result, false, res)
			return
		}

		res.decoded = int(oIdx)

		// Verify checksum
		if this.hasher != nil {
			checksum2 := this.hasher.Hash(this.data[0:res.decoded])

			if checksum2 != checksum1 {
				errMsg := fmt.Sprintf("Corrupted bitstream: expected checksum %x, found %x", checksum1, checksum2)
				res.err = NewIOError(errMsg, ERR_PROCESS_BLOCK)
				notify(nil, this.result, false, res)
				return
			}
		}

	}

	notify(nil, this.result, false, res)
}
Example #2
0
func (this *EncodingTask) encode() {
	transform, err := function.NewByteFunction(this.blockLength, this.typeOfTransform)

	if err != nil {
		<-this.input
		this.output <- NewIOError(err.Error(), ERR_CREATE_CODEC)
		return
	}

	buffer := this.buf
	requiredSize := transform.MaxEncodedLen(int(this.blockLength))

	if requiredSize == -1 {
		// Max size unknown => guess
		requiredSize = int(this.blockLength*5) >> 2
	}

	if this.typeOfTransform == function.NULL_TRANSFORM_TYPE {
		buffer = this.data // share buffers if no transform
	} else if len(buffer) < requiredSize {
		buffer = make([]byte, requiredSize)
	}

	mode := byte(0)
	dataSize := uint(0)
	postTransformLength := this.blockLength
	checksum := uint32(0)
	iIdx := uint(0)
	oIdx := uint(0)

	// Compute block checksum
	if this.hasher != nil {
		checksum = this.hasher.Hash(this.data[0:this.blockLength])
	}

	if len(this.listeners) > 0 {
		// Notify before transform
		evt := &BlockEvent{eventType: EVT_BEFORE_TRANSFORM, blockId: this.currentBlockId,
			blockSize: int(this.blockLength), hash: checksum, time_: time.Now(),
			hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	if this.blockLength <= SMALL_BLOCK_SIZE {
		// Just copy
		if !kanzi.SameByteSlices(buffer, this.data, false) {
			copy(buffer, this.data[0:this.blockLength])
		}

		iIdx += this.blockLength
		oIdx += this.blockLength
		mode = byte(SMALL_BLOCK_SIZE | (this.blockLength & COPY_LENGTH_MASK))
	} else {

		// Forward transform
		iIdx, oIdx, err = transform.Forward(this.data, buffer)

		if err != nil {
			// Transform failed (probably due to lack of space in output buffer)
			if !kanzi.SameByteSlices(buffer, this.data, false) {
				copy(buffer, this.data)
			}

			iIdx = this.blockLength
			oIdx = this.blockLength
			mode |= SKIP_FUNCTION_MASK
		}

		postTransformLength = oIdx

		for i := uint64(0xFF); i < uint64(postTransformLength); i <<= 8 {
			dataSize++
		}

		if dataSize > 3 {
			<-this.input
			this.output <- NewIOError("Invalid block data length", ERR_WRITE_FILE)
			return
		}

		// Record size of 'block size' - 1 in bytes
		mode |= byte(dataSize & 0x03)
		dataSize++
	}

	if len(this.listeners) > 0 {
		// Notify after transform
		evt := &BlockEvent{eventType: EVT_AFTER_TRANSFORM, blockId: this.currentBlockId,
			blockSize: int(postTransformLength), hash: checksum, time_: time.Now(),
			hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	// Wait for the concurrent task processing the previous block to complete
	// entropy encoding. Entropy encoding must happen sequentially (and
	// in the correct block order) in the bitstream.
	err2 := <-this.input

	if err2 != nil {
		this.output <- err2
		return
	}

	// Each block is encoded separately
	// Rebuild the entropy encoder to reset block statistics
	ee, err := entropy.NewEntropyEncoder(this.obs, this.typeOfEntropy)

	if err != nil {
		this.output <- NewIOError(err.Error(), ERR_CREATE_CODEC)
		return
	}

	// Write block 'header' (mode + compressed length)
	written := this.obs.Written()
	this.obs.WriteBits(uint64(mode), 8)

	if dataSize > 0 {
		this.obs.WriteBits(uint64(postTransformLength), 8*dataSize)
	}

	// Write checksum
	if this.hasher != nil {
		this.obs.WriteBits(uint64(checksum), 32)
	}

	if len(this.listeners) > 0 {
		// Notify before entropy
		evt := &BlockEvent{eventType: EVT_BEFORE_ENTROPY, blockId: this.currentBlockId,
			blockSize: int(postTransformLength), time_: time.Now(),
			hash: checksum, hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	// Entropy encode block
	_, err = ee.Encode(buffer[0:postTransformLength])

	if err != nil {
		this.output <- NewIOError(err.Error(), ERR_PROCESS_BLOCK)
		return
	}

	// Dispose before displaying statistics. Dispose may write to the bitstream
	ee.Dispose()

	if len(this.listeners) > 0 {
		// Notify after entropy
		evt := &BlockEvent{eventType: EVT_AFTER_ENTROPY, blockId: this.currentBlockId,
			blockSize: int((this.obs.Written() - written) / 8), time_: time.Now(),
			hash: checksum, hashing: this.hasher != nil}
		notifyListeners(this.listeners, evt)
	}

	// Notify of completion of the task
	this.output <- error(nil)
}