Beispiel #1
0
func (this *CompressedInputStream) readHeader() error {
	defer func() {
		if r := recover(); r != nil {
			panic(NewIOError("Cannot read bitstream header: "+r.(error).Error(), ERR_READ_FILE))
		}
	}()

	// Read stream type
	fileType := this.ibs.ReadBits(32)

	// Sanity check
	if fileType != BITSTREAM_TYPE {
		errMsg := fmt.Sprintf("Invalid stream type: expected %#x, got %#x", BITSTREAM_TYPE, fileType)
		return NewIOError(errMsg, ERR_INVALID_FILE)
	}

	version := this.ibs.ReadBits(7)

	// Sanity check
	if version != BITSTREAM_FORMAT_VERSION {
		errMsg := fmt.Sprintf("Invalid bitstream, cannot read this version of the stream: %d", version)
		return NewIOError(errMsg, ERR_STREAM_VERSION)
	}

	// Read block checksum
	if this.ibs.ReadBit() == 1 {
		var err error
		this.hasher, err = util.NewXXHash(BITSTREAM_TYPE)

		if err != nil {
			return err
		}
	}

	// Read entropy codec
	this.entropyType = byte(this.ibs.ReadBits(5))

	// Read transform
	this.transformType = byte(this.ibs.ReadBits(5))

	// Read block size
	this.blockSize = uint(this.ibs.ReadBits(26)) << 4

	if this.blockSize < MIN_BITSTREAM_BLOCK_SIZE || this.blockSize > MAX_BITSTREAM_BLOCK_SIZE {
		errMsg := fmt.Sprintf("Invalid bitstream, incorrect block size: %d", this.blockSize)
		return NewIOError(errMsg, ERR_BLOCK_SIZE)
	}

	// Read reserved bits
	this.ibs.ReadBits(4)

	if this.debugWriter != nil {
		fmt.Fprintf(this.debugWriter, "Checksum set to %v\n", (this.hasher != nil))
		fmt.Fprintf(this.debugWriter, "Block size set to %d bytes\n", this.blockSize)
		w1 := function.GetByteFunctionName(this.transformType)

		if w1 == "NONE" {
			w1 = "no"
		}

		fmt.Fprintf(this.debugWriter, "Using %v transform (stage 1)\n", w1)
		w2 := entropy.GetEntropyCodecName(this.entropyType)

		if w2 == "NONE" {
			w2 = "no"
		}

		fmt.Fprintf(this.debugWriter, "Using %v entropy codec (stage 2)\n", w2)
	}

	return nil
}
Beispiel #2
0
func main() {
	var filename = flag.String("input", "c:\\temp\\rt.jar", "name of the input file")
	flag.Parse()
	iter := 500
	fmt.Printf("Processing %v\n", *filename)
	fmt.Printf("%v iterations\n", iter)

	{
		fmt.Printf("XXHash speed test\n")
		file, err := os.Open(*filename)

		if err != nil {
			fmt.Printf("Cannot open %s", *filename)

			return
		}

		defer file.Close()
		buffer := make([]byte, 16384)
		before := time.Now()
		hash, err := util.NewXXHash(uint32(0))

		if err != nil {
			fmt.Printf("Failed to create hash: %v\n", err)
			return
		}

		length, err := file.Read(buffer)
		size := int64(0)
		res := uint32(0)

		for length > 0 {
			if err != nil {
				fmt.Printf("Failed to read the next chunk of input file '%v': %v\n", *filename, err)
				return
			}

			for i := 0; i < iter; i++ {
				res += hash.Hash(buffer[0:length])
			}

			size += int64(length * iter)
			length, err = file.Read(buffer)
		}

		after := time.Now()
		delta := after.Sub(before).Nanoseconds() / 1000000 // convert to ms
		fmt.Printf("XXHash res=%x\n", res)
		fmt.Printf("Elapsed [ms]: %v\n", delta)
		fmt.Printf("Throughput [MB/s]: %v\n", (size/1024*1000/1024)/delta)
	}

	fmt.Printf("\n")

	{
		fmt.Printf("MurmurHash3 speed test\n")
		file, err := os.Open(*filename)

		if err != nil {
			fmt.Printf("Cannot open %s", *filename)

			return
		}

		defer file.Close()
		buffer := make([]byte, 16384)
		before := time.Now()
		hash, err := util.NewMurMurHash3(uint32(0))

		if err != nil {
			fmt.Printf("Failed to create hash: %v\n", err)
			return
		}

		length, err := file.Read(buffer)
		size := int64(0)
		res := uint32(0)

		for length > 0 {
			if err != nil {
				fmt.Printf("Failed to read the next chunk of input file '%v': %v\n", *filename, err)
				return
			}

			for i := 0; i < iter; i++ {
				res += hash.Hash(buffer[0:length])
			}

			size += int64(length * iter)
			length, err = file.Read(buffer)
		}

		after := time.Now()
		delta := after.Sub(before).Nanoseconds() / 1000000 // convert to ms
		fmt.Printf("MurmurHash3 res=%x\n", res)
		fmt.Printf("Elapsed [ms]: %v\n", delta)
		fmt.Printf("Throughput [MB/s]: %v\n", (size/1024*1000/1024)/delta)
	}
}
Beispiel #3
0
func NewCompressedOutputStream(entropyCodec string, functionType string, os io.WriteCloser, blockSize uint,
	checksum bool, debugWriter io.Writer, jobs uint) (*CompressedOutputStream, error) {
	if os == nil {
		return nil, NewIOError("Invalid null output stream parameter", ERR_CREATE_STREAM)
	}

	if blockSize > MAX_BITSTREAM_BLOCK_SIZE {
		errMsg := fmt.Sprintf("The block size must be at most %d MB", MAX_BITSTREAM_BLOCK_SIZE>>20)
		return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
	}

	if blockSize < MIN_BITSTREAM_BLOCK_SIZE {
		errMsg := fmt.Sprintf("The block size must be at least %d", MIN_BITSTREAM_BLOCK_SIZE)
		return nil, NewIOError(errMsg, ERR_CREATE_STREAM)
	}

	if int(blockSize)&-16 != int(blockSize) {
		return nil, NewIOError("The block size must be a multiple of 16", ERR_CREATE_STREAM)
	}

	if jobs < 1 || jobs > 16 {
		return nil, NewIOError("The number of jobs must be in [1..16]", ERR_CREATE_STREAM)
	}

	this := new(CompressedOutputStream)
	var err error

	bufferSize := blockSize

	if bufferSize > 65536 {
		bufferSize = 65536
	}

	if this.obs, err = bitstream.NewDefaultOutputBitStream(os, bufferSize); err != nil {
		return nil, err
	}

	// Check entropy type validity (panic on error)
	this.entropyType = entropy.GetEntropyCodecType(entropyCodec)

	// Check transform type validity (panic on error)
	this.transformType = function.GetByteFunctionType(functionType)

	this.blockSize = blockSize

	if checksum == true {
		this.hasher, err = util.NewXXHash(BITSTREAM_TYPE)

		if err != nil {
			return nil, err
		}
	}

	this.data = make([]byte, jobs*blockSize)
	this.buffers = make([][]byte, jobs)

	for i := range this.buffers {
		this.buffers[i] = EMPTY_BYTE_SLICE
	}

	this.debugWriter = debugWriter
	this.jobs = int(jobs)
	this.blockId = 0
	this.channels = make([]chan error, this.jobs+1)

	for i := range this.channels {
		this.channels[i] = make(chan error)
	}

	this.listeners = make([]BlockListener, 0)
	return this, nil
}