util.go

// Copyright 2012, Kevin Ko <kevin@faveset.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rabin

import (
	Crand "crypto/rand"
	"math"
	"math/big"
	"math/rand"
)

// 8-bit log table.  Use an array to avoid bounds checking.
var logTable *[256]int8

type rabinRollingTables32 struct {
	// m is the rolling window size in bytes

	// t^{8m}
	t8m0 *[256]uint64
	// t^{8m + 8}
	t8m8 *[256]uint64
	// t^{8m + 16}
	t8m16 *[256]uint64
	// t^{8m + 24}
	t8m24 *[256]uint64
}

type rabinTables32 struct {
	// t64 is [0]
	raw *[4][256]uint64

	t64 *[256]uint64
	t72 *[256]uint64
	t80 *[256]uint64
	t88 *[256]uint64
}

type rabinTables64 struct {
	// t64 is [0]
	raw *[8][256]uint64

	t64  *[256]uint64
	t72  *[256]uint64
	t80  *[256]uint64
	t88  *[256]uint64
	t96  *[256]uint64
	t104 *[256]uint64
	t112 *[256]uint64
	t120 *[256]uint64
}

func init() {
	// Generated by make_log_table.py
	logTable = &[256]int8{
		-1, 0, 1, 1,
		2, 2, 2, 2,
		3, 3, 3, 3, 3, 3, 3, 3,
		4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
		5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
		5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
		6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
		6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
		6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
		6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
		7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
	}
}

func Log2(v uint64) uint64 {
	if hiWord := v >> 32; hiWord > 0 {
		return 32 + log2_32(logTable, hiWord)
	}
	return log2_32(logTable, v)
}

// v is a 32-bit word.  This returns log2 of v.
func log2_32(logTable *[256]int8, v uint64) uint64 {
	if hiWord := v >> 16; hiWord > 0 {
		if hiByte := hiWord >> 8; hiByte > 0 {
			return 24 + uint64(logTable[hiByte])
		}
		return 16 + uint64(logTable[hiWord])
	}

	if hiByte := v >> 8; hiByte > 0 {
		return 8 + uint64(logTable[hiByte])
	}
	return uint64(logTable[v])
}

// Use the Sieve of Eratosthenes to determine prime numbers less than n.
func CalcPrimes(n int) []int {
	primes := make([]int, 0)

	// First, test for evens.
	if n >= 2 {
		primes = append(primes, 2)
	}

	// Room for all odd integers between 3 and n, inclusive.
	numCandidates := (n-3+1)/2 + 1
	candidates := make([]bool, numCandidates)

	// Candidates start from value 3 and are odd.
	getIndex := func(value int) int {
		return (value - 3) / 2
	}
	getValue := func(index int) int {
		return 2*index + 3
	}

	for ii := 0; getValue(ii) <= n; ii++ {
		candidates[ii] = true
	}

	p := 3
	for {
		for v := p * p; v <= n; v += 2 * p {
			// Mark the multiples greater than p.
			index := getIndex(v)
			candidates[index] = false
		}

		done := true

		// Find the next starting value for p that is greater than p.
		for ii := getIndex(p) + 1; ii < numCandidates; ii++ {
			if candidates[ii] {
				p = getValue(ii)
				done = false
				break
			}
		}
		if done {
			break
		}
	}

	// Tabulate the results.
	for ii := 0; ii < numCandidates; ii++ {
		if candidates[ii] {
			primes = append(primes, getValue(ii))
		}
	}

	return primes
}

// Returns true if v is a power of 2.
func IsPowerOfTwo(v int) bool {
	return (v & (v - 1)) == 0
}

func MakeRandom(gen *rand.Rand, degree int) *Polynomial {
	if degree == 0 {
		return NewPolynomialFromInt(gen.Int63n(2))
	}

	coeffs := new(big.Int)
	// x^0 + x^1 + ... + x^n => n + 1 terms
	// However, only the first n terms are variable.  (x^n is fixed to
	// have degree n.)  Thus, we randomly generate the first n terms
	// and fix the final term x^n.
	numBits := degree
	numBlocks := numBits / 32
	for ii := 0; ii < numBlocks; ii++ {
		v := gen.Uint32()

		// Merge.
		bigV := big.NewInt(int64(v))
		coeffs.Lsh(coeffs, 32).Or(coeffs, bigV)
	}
	// Handle the remainder.
	numRemainingBits := uint(numBits % 32)
	if numRemainingBits > 0 {
		mask := (int64(1) << numRemainingBits) - 1
		v := int64(gen.Uint32()) & mask
		coeffs.Lsh(coeffs, numRemainingBits).Or(coeffs, big.NewInt(v))
	}
	coeffs.SetBit(coeffs, degree, 1)
	return NewPolynomial(uint(degree), coeffs)
}

// Returns an irreducible polynomial of given degree.
func FindIrreducible(degree int) *Polynomial {
	// Seed the source with a strongly random seed (crypto/rand).
	maxSeed := big.NewInt(math.MaxInt64)
	seed, err := Crand.Int(Crand.Reader, maxSeed)
	if err != nil {
		panic(err)
	}
	source := rand.NewSource(seed.Int64())
	randGen := rand.New(source)

	for {
		p := MakeRandom(randGen, degree)
		if p.Irreducible() {
			return p
		}
	}
}

// Calculates and returns b(t) t^{offsetDegree} mod P(t), which is
// the fingerprint f of b.
func calcTableValue(offsetDegree, b int, p *Polynomial) (f uint64) {
	bigCoeffs := big.NewInt(int64(b))
	bigCoeffs.Lsh(bigCoeffs, uint(offsetDegree))
	tmp := NewPolynomialFromBigInt(bigCoeffs)
	tmp.Mod(tmp, p)
	_, f = tmp.Uint64()
	return
}

// Naively fingerprints the given data using polynomials.
func RabinFingerprint(p *Polynomial, data []byte) *Polynomial {
	// Convert data to a polynomial.
	coeffs := new(big.Int).SetBytes(data)
	dataPoly := NewPolynomialFromBigInt(coeffs)
	return dataPoly.Mod(dataPoly, p)
}

// Uses a fixed degree 64 polynomial.
func RabinFingerprintFixed(data []byte) (fp uint64) {
	p := NewPolynomialFromUint64(kIrreduciblePolyDegree, kIrreduciblePolyCoeffs)
	fpPoly := RabinFingerprint(p, data)
	_, fp = fpPoly.Uint64()
	return
}

// Returns a 64-entry table of t^k mod P(t) for
// basePower \le k < (64 + basePower).
//
// basePower must be >= 64.
//
// P(t) is kIrreduciblePolyCoeffs
// See rabin.tex (Basic Operations) for an explanation.
func makePowerTable(basePower int) *[64]uint64 {
	powerTable := &[64]uint64{}

	// t^k mod P(t) = P(t) + t^k.  Note that degree(P(t)) = 64 and that
	// kIrreduciblePolyCoeffs does not include the implied t^64 term.
	pTk := uint64(kIrreduciblePolyCoeffs)

	// We always start from t^64.  Polynomials of degree < 64 are trivially
	// determined, since deg(P(t)) = 64.
	index := 64
	curr := pTk

	// Advance to basePower.
	for index < basePower {
		prev := curr

		msb := prev >> 63
		// multiply by t (shifting out the MSB)
		curr = prev << 1
		if msb > 0 {
			curr ^= pTk
		}
		index++
	}

	powerTable[0] = curr
	for ii := 1; ii < len(powerTable); ii++ {
		p0 := powerTable[ii-1]
		p0Msb := p0 >> 63
		// multiply by t (shifting out the MSB)
		p := p0 << 1
		if p0Msb > 0 {
			p ^= pTk
		}
		powerTable[ii] = p
	}
	return powerTable
}

// Generates byte tables for a 32-bit word using the given power table:
//
//   (b_1 t^24 + b_2 t^16 + b_3 t^8 + b_4) t^basePower
//
// The generated tables can be used to determine the terms:
//    b_1 t^{24 + basePower}
//    b_2 t^{16 + basePower}
//    b_3 t^{8 + basePower}
//    b_4 t^{basePower}
// for all b_i.
//
// tables[0] will correspond with t^{basePower}.
func makeTables32Raw(powerTable *[64]uint64) (tables *[4][256]uint64) {
	tables = &[4][256]uint64{}
	for ii := 0; ii < 256; ii++ {
		// Expand ii bit-wise.
		for jj := 0; jj < 8; jj++ {
			isBitSet := (ii >> uint(jj)) & 0x1
			if isBitSet == 0 {
				continue
			}

			// Fill by each table offset.
			for kk := 0; kk < 4; kk++ {
				// Expand bit-wise:
				// ii_1 t^{basePower + 7} + ... ii_8 t^{basePower}
				tables[kk][ii] ^= powerTable[8*kk+jj]
			}
		}
	}
	return tables
}

func makeRabinTables32() *rabinTables32 {
	rawTables := makeRabinTables32Raw()
	return &rabinTables32{
		raw: rawTables,
		t64: &rawTables[0],
		t72: &rawTables[1],
		t80: &rawTables[2],
		t88: &rawTables[3],
	}
}

// windowSize is in bytes.
func makeRabinRollingTables32(windowSize int) *rabinRollingTables32 {
	rawTables := makeRabinRollingTables32Raw(windowSize)
	return &rabinRollingTables32{
		t8m0:  &rawTables[0],
		t8m8:  &rawTables[1],
		t8m16: &rawTables[2],
		t8m24: &rawTables[3],
	}
}

// windowSize is the number of bytes for the window.  This generates 4
// tables for a 32-bit word starting at t^{8*size}.
func makeRabinRollingTables32Raw(windowSize int) (tables *[4][256]uint64) {
	powerTable := makePowerTable(8 * windowSize)
	return makeTables32Raw(powerTable)
}

// T64 is [0].
func makeRabinTables32Raw() *[4][256]uint64 {
	powerTable := makePowerTable(64)
	return makeTables32Raw(powerTable)
}

func makeRabinTables64() *rabinTables64 {
	rawTables := makeRabinTables64Raw()
	return &rabinTables64{
		raw:  rawTables,
		t64:  &rawTables[0],
		t72:  &rawTables[1],
		t80:  &rawTables[2],
		t88:  &rawTables[3],
		t96:  &rawTables[4],
		t104: &rawTables[5],
		t112: &rawTables[6],
		t120: &rawTables[7],
	}
}

// T64 is [0]
func makeRabinTables64Raw() (tables *[8][256]uint64) {
	powerTable := makePowerTable(64)

	tables = &[8][256]uint64{}
	for ii := 0; ii < 256; ii++ {
		// Expand ii bit-wise.
		for jj := 0; jj < 8; jj++ {
			isBitSet := (ii >> uint(jj)) & 0x1
			if isBitSet == 0 {
				continue
			}

			// Fill by each table offset.
			for kk := 0; kk < 8; kk++ {
				// (ii) * t^{basePower}
				basePower := 64 + 8*kk

				// Expand bit-wise:
				// ii_1 t^{basePower + 7} + ... ii_8 t^{basePower}
				powerOffset := basePower + jj - 64
				tables[kk][ii] ^= powerTable[powerOffset]
			}
		}
	}
	return tables
}

// p is the irreducible polynomial.  This generates the 4 tables
// TA, TB, TC, TD for fast 32-bit Rabin fingerprinting.  (See rabin.tex.)
//
// Note that tables[0] = TA, tables[1] = TB, etc.
func MakeRabinTables32FromPoly(p *Polynomial) (tables *[4][256]uint64) {
	tables = &[4][256]uint64{}

	for i := 0; i < 256; i++ {
		for j := 0; j < 4; j++ {
			// Order so that T88 is tables[3].
			tables[j][i] = calcTableValue(64+8*j, i, p)
		}
	}
	return
}

// p is the irreducible polynomial.  This generates the 8 tables
// TA, TB, TC, TD, TE, TF, TG, TH for fast 64-bit Rabin fingerprinting.
// (See rabin.tex.)
//
// Note that tables[0] = TA, tables[1] = TB, etc.
func MakeRabinTables64FromPoly(p *Polynomial) (tables *[8][256]uint64) {
	tables = &[8][256]uint64{}

	for i := 0; i < 256; i++ {
		for j := 0; j < 8; j++ {
			tables[j][i] = calcTableValue(64+8*j, i, p)
		}
	}
	return
}