Example #1
0
func NewBlockCompressor() (*BlockCompressor, error) {
	this := new(BlockCompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Bool("verbose", false, "display the block size at each stage (in bytes, floor rounding if fractional)")
	var silent = flag.Bool("silent", false, "silent mode, no output (except warnings and errors)")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var inputName = flag.String("input", "", "mandatory name of the input file to encode")
	var outputName = flag.String("output", "", "optional name of the output file (defaults to <input.knz>), or 'none' for dry-run")
	var blockSize = flag.String("block", "1048576", "size of the input blocks, multiple of 8, max 512 MB (depends on transform), min 1KB, default 1MB")
	var entropy = flag.String("entropy", "Huffman", "entropy codec to use [None|Huffman*|ANS|Range|PAQ|FPAQ|CM]")
	var function = flag.String("transform", "BWT+MTF", "transform to use [None|BWT|BWTS|Snappy|LZ4|RLT]")
	var cksum = flag.Bool("checksum", false, "enable block checksum")
	var tasks = flag.Int("jobs", 1, "number of concurrent jobs")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help                : display this message", true)
		printOut("-verbose             : display the block size at each stage (in bytes, floor rounding if fractional)", true)
		printOut("-silent              : silent mode, no output (except warnings and errors)", true)
		printOut("-overwrite           : overwrite the output file if it already exists", true)
		printOut("-input=<inputName>   : mandatory name of the input file to encode", true)
		printOut("-output=<outputName> : optional name of the output file (defaults to <input.knz>) or 'none' for dry-run", true)
		printOut("-block=<size>        : size of the input blocks, multiple of 8, max 512 MB (depends on transform), min 1KB, default 1MB", true)
		printOut("-entropy=<codec>     : entropy codec to use [None|Huffman*|ANS|Range|PAQ|FPAQ|CM]", true)
		printOut("-transform=<codec>   : transform to use [None|BWT*|BWTS|Snappy|LZ4|RLT]", true)
		printOut("                       for BWT(S), an optional GST can be provided: [MTF|RANK|TIMESTAMP]", true)
		printOut("                       EG: BWT+RANK or BWTS+MTF (default is BWT+MTF)", true)
		printOut("-checksum            : enable block checksum", true)
		printOut("-jobs=<jobs>         : number of concurrent jobs", true)
		printOut("", true)
		printOut("EG. go run BlockCompressor -input=foo.txt -output=foo.knz -overwrite -transform=BWT+MTF -block=4m -entropy=FPAQ -verbose -jobs=4", true)
		os.Exit(0)
	}

	if *silent == true && *verbose == true {
		printOut("Warning: both 'silent' and 'verbose' options were selected, ignoring 'verbose'", true)
		*verbose = false
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(io.ERR_MISSING_FILENAME)
	}

	if len(*outputName) == 0 {
		*outputName = *inputName + ".knz"
	}

	this.verbose = *verbose
	this.silent = *silent
	this.overwrite = *overwrite
	this.inputName = *inputName
	this.outputName = *outputName
	strBlockSize := strings.ToUpper(*blockSize)

	// Process K or M suffix
	scale := 1

	if strBlockSize[len(strBlockSize)-1] == 'K' {
		strBlockSize = strBlockSize[0 : len(strBlockSize)-1]
		scale = 1024
	} else if strBlockSize[len(strBlockSize)-1] == 'M' {
		strBlockSize = strBlockSize[0 : len(strBlockSize)-1]
		scale = 1024 * 1024
	}

	bSize, err := strconv.Atoi(strBlockSize)

	if err != nil {
		fmt.Printf("Invalid block size provided on command line: %v\n", *blockSize)
		os.Exit(io.ERR_BLOCK_SIZE)
	}

	this.blockSize = uint(scale * bSize)
	this.entropyCodec = strings.ToUpper(*entropy)
	this.transform = strings.ToUpper(*function)
	this.checksum = *cksum
	this.jobs = uint(*tasks)
	this.listeners = list.New()

	if this.verbose == true {
		listener, _ := io.NewInfoPrinter(io.ENCODING, os.Stdout)
		this.listeners.PushFront(listener)
	}

	return this, nil
}
Example #2
0
func NewBlockDecompressor() (*BlockDecompressor, error) {
	this := new(BlockDecompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Int("verbose", 1, "set the verbosity level [0..4]")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var inputName = flag.String("input", "", "mandatory name of the input file to decode")
	var outputName = flag.String("output", "", "optional name of the output file or 'none' for dry-run")
	var tasks = flag.Int("jobs", 1, "number of concurrent jobs")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help                : display this message", true)
		printOut("-verbose=<level>     : set the verbosity level [0..4]", true)
		printOut("                       0=silent, 1=default, 2=display block size (byte rounded)", true)
		printOut("                       3=display timings, 4=display extra information", true)
		printOut("-overwrite           : overwrite the output file if it already exists", true)
		printOut("-input=<inputName>   : mandatory name of the input file to decode", true)
		printOut("-output=<outputName> : optional name of the output file or 'none' for dry-run", true)
		printOut("-jobs=<jobs>         : number of concurrent jobs", true)
		printOut("", true)
		printOut("EG. go run BlockDecompressor -input=foo.knz -overwrite -verbose=2 -jobs=2", true)
		os.Exit(0)
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(kio.ERR_MISSING_PARAM)
	}

	if strings.HasSuffix(*inputName, ".knz") == false {
		printOut("Warning: the input file name does not end with the .KNZ extension", true)
	}

	if len(*outputName) == 0 {
		if strings.HasSuffix(*inputName, ".knz") == false {
			*outputName = *inputName + ".tmp"
		} else {
			*outputName = strings.TrimRight(*inputName, ".knz")
		}
	}

	if *tasks < 1 {
		fmt.Printf("Invalid number of jobs provided on command line: %v\n", *tasks)
		os.Exit(kio.ERR_INVALID_PARAM)
	}

	if *verbose < 0 {
		fmt.Printf("Invalid verbosity level provided on command line: %v\n", *verbose)
		os.Exit(kio.ERR_INVALID_PARAM)
	}

	this.verbosity = uint(*verbose)
	this.inputName = *inputName
	this.outputName = *outputName
	this.overwrite = *overwrite
	this.jobs = uint(*tasks)
	this.listeners = make([]kio.BlockListener, 0)

	if this.verbosity > 1 {
		if listener, err := kio.NewInfoPrinter(this.verbosity, kio.DECODING, os.Stdout); err == nil {
			this.AddListener(listener)
		}
	}

	return this, nil
}
Example #3
0
func NewBlockDecompressor() (*BlockDecompressor, error) {
	this := new(BlockDecompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Bool("verbose", false, "display the block size at each stage (in bytes, floor rounding if fractional)")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var silent = flag.Bool("silent", false, "silent mode, no output (except warnings and errors)")
	var inputName = flag.String("input", "", "mandatory name of the input file to decode")
	var outputName = flag.String("output", "", "optional name of the output file or 'none' for dry-run")
	var tasks = flag.Int("jobs", 1, "number of concurrent jobs")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help                : display this message", true)
		printOut("-verbose             : display the block size at each stage (in bytes, floor rounding if fractional)", true)
		printOut("-overwrite           : overwrite the output file if it already exists", true)
		printOut("-silent              : silent mode, no output (except warnings and errors)", true)
		printOut("-input=<inputName>   : mandatory name of the input file to decode", true)
		printOut("-output=<outputName> : optional name of the output file or 'none' for dry-run", true)
		printOut("-jobs=<jobs>         : number of concurrent jobs", true)
		printOut("", true)
		printOut("EG. go run BlockDecompressor -input=foo.knz -overwrite -verbose -jobs=2", true)
		os.Exit(0)
	}

	if *silent == true && *verbose == true {
		printOut("Warning: both 'silent' and 'verbose' options were selected, ignoring 'verbose'", true)
		*verbose = false
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(io.ERR_MISSING_FILENAME)
	}

	if strings.HasSuffix(*inputName, ".knz") == false {
		printOut("Warning: the input file name does not end with the .KNZ extension", true)
	}

	if len(*outputName) == 0 {
		if strings.HasSuffix(*inputName, ".knz") == false {
			*outputName = *inputName + ".tmp"
		} else {
			*outputName = strings.TrimRight(*inputName, ".knz")
		}
	}

	this.verbose = *verbose
	this.silent = *silent
	this.inputName = *inputName
	this.outputName = *outputName
	this.overwrite = *overwrite
	this.jobs = uint(*tasks)
	this.listeners = list.New()

	if this.verbose == true {
		listener, _ := io.NewInfoPrinter(io.DECODING, os.Stdout)
		this.listeners.PushFront(listener)
	}

	return this, nil
}
Example #4
0
func NewBlockCompressor() (*BlockCompressor, error) {
	this := new(BlockCompressor)

	// Define flags
	var help = flag.Bool("help", false, "display the help message")
	var verbose = flag.Int("verbose", 1, "set the verbosity level [0..4]")
	var overwrite = flag.Bool("overwrite", false, "overwrite the output file if it already exists")
	var inputName = flag.String("input", "", "mandatory name of the input file to encode")
	var outputName = flag.String("output", "", "optional name of the output file (defaults to <input.knz>), or 'none' for dry-run")
	var blockSize = flag.String("block", "1048576", "size of the input blocks, multiple of 16, max 512 MB (depends on transform), min 1KB, default 1MB")
	var entropy = flag.String("entropy", "Huffman", "entropy codec to use [None|Huffman*|ANS|Range|PAQ|FPAQ|CM]")
	var function = flag.String("transform", "BWT+MTF", "transform to use [None|BWT|BWTS|Snappy|LZ4|RLT]")
	var cksum = flag.Bool("checksum", false, "enable block checksum")
	var tasks = flag.Int("jobs", 1, "number of concurrent jobs")

	// Parse
	flag.Parse()

	if *help == true {
		printOut("-help                : display this message", true)
		printOut("-verbose=<level>     : set the verbosity level [0..4]", true)
		printOut("                       0=silent, 1=default, 2=display block size (byte rounded)", true)
		printOut("                       3=display timings, 4=display extra information", true)
		printOut("-overwrite           : overwrite the output file if it already exists", true)
		printOut("-input=<inputName>   : mandatory name of the input file to encode", true)
		printOut("-output=<outputName> : optional name of the output file (defaults to <input.knz>) or 'none' for dry-run", true)
		printOut("-block=<size>        : size of the input blocks, multiple of 16, max 1 GB (transform dependent), min 1 KB, default 1 MB", true)
		printOut("-entropy=<codec>     : entropy codec to use [None|Huffman*|ANS|Range|PAQ|FPAQ|TPAQ|CM]", true)
		printOut("-transform=<codec>   : transform to use [None|BWT*|BWTS|Snappy|LZ4|RLT]", true)
		printOut("                       for BWT(S), an optional GST can be provided: [MTF|RANK|TIMESTAMP]", true)
		printOut("                       EG: BWT+RANK or BWTS+MTF (default is BWT+MTF)", true)
		printOut("-checksum            : enable block checksum", true)
		printOut("-jobs=<jobs>         : number of concurrent jobs", true)
		printOut("", true)
		printOut("EG. go run BlockCompressor -input=foo.txt -output=foo.knz -overwrite -transform=BWT+MTF -block=4m -entropy=FPAQ -verbose=2 -jobs=4", true)
		os.Exit(0)
	}

	if len(*inputName) == 0 {
		fmt.Printf("Missing input file name, exiting ...\n")
		os.Exit(kio.ERR_MISSING_PARAM)
	}

	if len(*outputName) == 0 {
		*outputName = *inputName + ".knz"
	}

	if *tasks < 1 {
		fmt.Printf("Invalid number of jobs provided on command line: %v\n", *tasks)
		os.Exit(kio.ERR_INVALID_PARAM)
	}

	if *verbose < 0 {
		fmt.Printf("Invalid verbosity level provided on command line: %v\n", *verbose)
		os.Exit(kio.ERR_INVALID_PARAM)
	}

	this.verbosity = uint(*verbose)
	this.overwrite = *overwrite
	this.inputName = *inputName
	this.outputName = *outputName
	strBlockSize := strings.ToUpper(*blockSize)

	// Process K or M suffix
	scale := 1
	lastChar := strBlockSize[len(strBlockSize)-1]

	if lastChar == 'K' {
		strBlockSize = strBlockSize[0 : len(strBlockSize)-1]
		scale = 1024
	} else if lastChar == 'M' {
		strBlockSize = strBlockSize[0 : len(strBlockSize)-1]
		scale = 1024 * 1024
	} else if lastChar == 'G' {
		strBlockSize = strBlockSize[0 : len(strBlockSize)-1]
		scale = 1024 * 1024 * 1024
	}

	bSize, err := strconv.Atoi(strBlockSize)

	if err != nil || bSize <= 0 {
		fmt.Printf("Invalid block size provided on command line: %v\n", *blockSize)
		os.Exit(kio.ERR_BLOCK_SIZE)
	}

	this.blockSize = uint(scale * bSize)
	this.entropyCodec = strings.ToUpper(*entropy)
	this.transform = strings.ToUpper(*function)
	this.checksum = *cksum
	this.jobs = uint(*tasks)
	this.listeners = make([]kio.BlockListener, 0)

	if this.verbosity > 1 {
		if listener, err := kio.NewInfoPrinter(this.verbosity, kio.ENCODING, os.Stdout); err == nil {
			this.AddListener(listener)
		}
	}

	return this, nil
}