func GetHash(a string) (hash.Hash, error) {
var h hash.Hash
switch a {
case "adler32":
h = adler32.New()
case "crc32", "crc32ieee":
h = crc32.New(crc32.MakeTable(crc32.IEEE))
case "crc32castagnoli":
h = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case "crc32koopman":
h = crc32.New(crc32.MakeTable(crc32.Koopman))
case "crc64", "crc64iso":
h = crc64.New(crc64.MakeTable(crc64.ISO))
case "crc64ecma":
h = crc64.New(crc64.MakeTable(crc64.ECMA))
case "fnv", "fnv32":
h = fnv.New32()
case "fnv32a":
h = fnv.New32a()
case "fnv64":
h = fnv.New64()
case "fnv64a":
h = fnv.New64a()
case "hmac", "hmacsha256":
h = hmac.New(sha256.New, []byte(key))
case "hmacmd5":
h = hmac.New(md5.New, []byte(key))
case "hmacsha1":
h = hmac.New(sha1.New, []byte(key))
case "hmacsha512":
h = hmac.New(sha512.New, []byte(key))
case "md4":
h = md4.New()
case "md5":
h = md5.New()
case "ripemd160":
h = ripemd160.New()
case "sha1":
h = sha1.New()
case "sha224":
h = sha256.New224()
case "sha256":
h = sha256.New()
case "sha384":
h = sha512.New384()
case "sha512":
h = sha512.New()
default:
return nil, errors.New("Invalid algorithm")
}
return h, nil
}
func makeHash(name string) hash.Hash {
switch strings.ToLower(name) {
case "ripemd160":
return ripemd160.New()
case "md4":
return md4.New()
case "md5":
return md5.New()
case "sha1":
return sha1.New()
case "sha256":
return sha256.New()
case "sha384":
return sha512.New384()
case "sha3-224":
return sha3.New224()
case "sha3-256":
return sha3.New256()
case "sha3-384":
return sha3.New384()
case "sha3-512":
return sha3.New512()
case "sha512":
return sha512.New()
case "sha512-224":
return sha512.New512_224()
case "sha512-256":
return sha512.New512_256()
case "crc32-ieee":
return crc32.NewIEEE()
case "crc64-iso":
return crc64.New(crc64.MakeTable(crc64.ISO))
case "crc64-ecma":
return crc64.New(crc64.MakeTable(crc64.ECMA))
case "adler32":
return adler32.New()
case "fnv32":
return fnv.New32()
case "fnv32a":
return fnv.New32a()
case "fnv64":
return fnv.New64()
case "fnv64a":
return fnv.New64a()
case "xor8":
return new(xor8)
case "fletch16":
return &fletch16{}
}
return nil
}
/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
func decStreamHeader(s *xzDec) xzRet {
if string(s.temp.buf[:len(headerMagic)]) != headerMagic {
return xzFormatError
}
if xzCRC32(s.temp.buf[len(headerMagic):len(headerMagic)+2], 0) !=
getLE32(s.temp.buf[len(headerMagic)+2:]) {
return xzDataError
}
if s.temp.buf[len(headerMagic)] != 0 {
return xzOptionsError
}
/*
* Of integrity checks, we support none (Check ID = 0),
* CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10)
* However, we will accept other check types too, but then the check
* won't be verified and a warning (xzUnsupportedCheck) will be given.
*/
s.checkType = xzCheck(s.temp.buf[len(headerMagic)+1])
if s.checkType > xzCheckMax {
return xzOptionsError
}
switch s.checkType {
case xzCheckNone:
// xzCheckNone: no action needed
case xzCheckCRC32:
s.check = crc32.New(xzCRC32Table)
case xzCheckCRC64:
s.check = crc64.New(xzCRC64Table)
case xzCheckSHA256:
s.check = sha256.New()
default:
return xzUnsupportedCheck
}
return xzOK
}
func NewSpecialCollectionCache(intent *intents.Intent, demux *Demultiplexer) *SpecialCollectionCache {
return &SpecialCollectionCache{
Intent: intent,
Demux: demux,
hash: crc64.New(crc64.MakeTable(crc64.ECMA)),
}
}
func (p *onePeer) UniqID() uint64 {
h := crc64.New(crctab)
h.Write(p.Ip6[:])
h.Write(p.Ip4[:])
h.Write([]byte{byte(p.Port >> 8), byte(p.Port)})
return h.Sum64()
}
func hashcrc64(s []byte) []byte {
var tab = crc64.MakeTable(crc64.ECMA)
h := crc64.New(tab)
h.Write(s)
return h.Sum(nil)
}
func (e *Engine) crc64_ecma() error {
data, err := computeHash(crc64.New(crc64.MakeTable(crc64.ECMA)), e.stack.Pop())
if err == nil {
e.stack.Push(data)
}
return err
}
func (a *Archiver) archiveWriter() error {
hash := crc64.New(crc64.MakeTable(crc64.ECMA))
output := io.MultiWriter(a.output, hash)
blockCount := 0
_, err := output.Write(fastArchiverHeader)
if err != nil {
return err
}
for block := range a.blockQueue {
err = block.writeBlock(output)
blockCount += 1
if err == nil && (blockCount%1000) == 0 {
err = writeChecksumBlock(hash, output)
}
if err != nil {
return err
}
}
return writeChecksumBlock(hash, output)
}
func process_file(filename string, complete chan Sumlist) {
sumlist := Sumlist{}
sumlist.filename = filename
// Open the file and bail if we fail
infile, err := os.Open(filename)
if err != nil {
log.Printf("Unable to open %s: %s", filename, err)
complete <- sumlist
return
}
defer infile.Close()
// Create the checksum objects
if flag_crc32 {
sumlist.sums = append(sumlist.sums, Checksum{"CRC32", crc32.New(crc32.IEEETable)})
}
if flag_crc64 {
sumlist.sums = append(sumlist.sums, Checksum{"CRC64", crc64.New(crc64.MakeTable(crc64.ISO))})
}
if flag_sha224 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA224", sha256.New224()})
}
if flag_sha256 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA256", sha256.New()})
}
if flag_sha384 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA384", sha512.New384()})
}
if flag_sha512 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA512", sha512.New()})
}
// Create our file reader
reader := bufio.NewReader(infile)
// Start a buffer and loop to read the entire file
buf := make([]byte, 4096)
for {
read_count, err := reader.Read(buf)
// If we get an error that is not EOF, then we have a problem
if err != nil && err != io.EOF {
log.Printf("Unable to open %s: %s", filename, err)
complete <- sumlist
return
}
// If the returned size is zero, we're at the end of the file
if read_count == 0 {
break
}
// Add the buffer contents to the checksum calculation
for _, sum := range sumlist.sums {
sum.hashFunc.Write(buf[:read_count])
}
}
complete <- sumlist
}
func file(fn string) {
ext := path.Ext(fn)
if ext == ".go" {
return
}
in, err := ioutil.ReadFile(fn)
if err != nil {
panic(err)
}
out, err := os.Create(fn + ".go")
if err != nil {
panic(err)
}
defer out.Close()
fmt.Fprintf(out, `package resource
func init() {
Resource[%q] = []byte{`, filepath.Base(fn))
for i, b := range in {
if i == 0 {
fmt.Fprintf(out, `%d`, b)
} else {
fmt.Fprintf(out, `, %d`, b)
}
}
hash := crc64.New(crc64.MakeTable(crc64.ISO))
hash.Write(in)
fmt.Fprintf(out, "}\n\tHash[%q] = \"W/\\\"%d\\\"\"\n}\n", filepath.Base(fn), hash.Sum64())
}
func TestUseCrc(t *testing.T) {
table := crc64.MakeTable(crc64.ISO)
h := crc64.New(table)
input := "someUniqueId"
io.WriteString(h, input)
sum1 := h.Sum64()
h.Reset()
input = "someOtherId"
io.WriteString(h, input)
sum2 := h.Sum64()
if sum1 == sum2 {
t.Errorf("Sums shouldn't match [%x] [%x]\n", sum1, sum2)
t.Fail()
return
}
h.Reset()
input = "someUniqueId"
io.WriteString(h, input)
sum3 := h.Sum64()
if sum1 != sum3 {
t.Errorf("Sums should match [%x] [%x]\n", sum1, sum3)
t.Fail()
return
}
}
func init() {
// Create the hashing function.
hasher := crc64.New(crc64.MakeTable(crc64.ECMA))
h := func(s string) uint64 {
hasher.Reset()
hasher.Write([]byte(s))
return hasher.Sum64()
}
// Get the current user name.
osU, err := user.Current()
u := "UNKNOW"
if err == nil {
u = osU.Username
}
// Create the constant to make build a unique ID.
start := uint64(time.Now().UnixNano())
user := h(u)
pid := uint64(os.Getpid())
// Initialize the channel and blank node type.
nextVal, tBlank = make(chan string, chanSize), Type("/_")
enc := base64.StdEncoding
go func() {
cnt := uint64(0)
for {
bs := []byte(fmt.Sprintf("%x:%x:%x:%x", start, user, pid, cnt))
nextVal <- enc.EncodeToString(bs)
cnt++
}
}()
}
func (h *HashTable) hash(k []byte) uint64 {
hash := crc64.New(crc64.MakeTable(crc64.ISO))
_, _ = hash.Write(k)
return hash.Sum64() % uint64(len(h.data))
}
func main() {
m := map[string]hash.Hash{
"had": adler32.New(),
"hc32": crc32.NewIEEE(),
"hc64e": crc64.New(crc64.MakeTable(crc64.ECMA)),
"hc64i": crc64.New(crc64.MakeTable(crc64.ISO)),
"hf32": fnv.New32(),
"hf32a": fnv.New32a(),
"hf64": fnv.New64(),
"hf64a": fnv.New64a(),
}
for n, h := range m {
runtime.GC()
testHash(n, h)
}
}
func (b *builder) buildTrie(t *Trie) uint64 {
n := t.root
// Get the ASCII offset. For the first trie, the ASCII block will be at
// position 0.
hasher := crc64.New(crcTable)
binary.Write(hasher, binary.BigEndian, n.values)
hash := hasher.Sum64()
v, ok := b.asciiBlockIdx[hash]
if !ok {
v = len(b.ValueBlocks)
b.asciiBlockIdx[hash] = v
b.ValueBlocks = append(b.ValueBlocks, n.values[:blockSize], n.values[blockSize:])
if v == 0 {
// Add the zero block at position 2 so that it will be assigned a
// zero reference in the lookup blocks.
// TODO: always do this? This would allow us to remove a check from
// the trie lookup, but at the expense of extra space. Analyze
// performance for unicode/norm.
b.ValueBlocks = append(b.ValueBlocks, make([]uint64, blockSize))
}
}
t.ASCIIIndex = v
// Compute remaining offsets.
t.Checksum = b.computeOffsets(n, true)
// We already subtracted the normal blockOffset from the index. Subtract the
// difference for starter bytes.
t.StarterIndex = n.index.index - (rootBlockOffset - blockOffset)
return t.Checksum
}
func main() {
var ECMATable = crc64.MakeTable(crc64.ECMA)
h := crc64.New(ECMATable)
h.Write([]byte("test"))
v := h.Sum64()
fmt.Println(v)
}
func (b *builder) computeOffsets(n *node, root bool) uint64 {
// For the first trie, the root lookup block will be at position 3, which is
// the offset for UTF-8 non-ASCII starter bytes.
first := len(b.IndexBlocks) == rootBlockOffset
if first {
b.IndexBlocks = append(b.IndexBlocks, n)
}
// We special-case the cases where all values recursively are 0. This allows
// for the use of a zero block to which all such values can be directed.
hash := uint64(0)
if n.children != nil || n.values != nil {
hasher := crc64.New(crcTable)
for _, c := range n.children {
var v uint64
if c != nil {
v = b.computeOffsets(c, false)
}
binary.Write(hasher, binary.BigEndian, v)
}
binary.Write(hasher, binary.BigEndian, n.values)
hash = hasher.Sum64()
}
if first {
b.indexBlockIdx[hash] = rootBlockOffset - blockOffset
}
// Compacters don't apply to internal nodes.
if n.children != nil {
v, ok := b.indexBlockIdx[hash]
if !ok {
v = len(b.IndexBlocks) - blockOffset
b.IndexBlocks = append(b.IndexBlocks, n)
b.indexBlockIdx[hash] = v
}
n.index = nodeIndex{0, v}
} else {
h, ok := b.valueBlockIdx[hash]
if !ok {
bestI, bestSize := 0, blockSize*b.ValueSize
for i, c := range b.Compactions[1:] {
if sz, ok := c.c.Size(n.values); ok && bestSize > sz {
bestI, bestSize = i+1, sz
}
}
c := &b.Compactions[bestI]
c.totalSize += bestSize
v := c.c.Store(n.values)
if c.maxHandle < v {
c.maxHandle = v
}
h = nodeIndex{bestI, int(v)}
b.valueBlockIdx[hash] = h
}
n.index = h
}
return hash
}
func NewReader(in io.Reader, enableCRC bool) *Reader {
r := new(Reader)
r.in = in
if enableCRC {
r.crc = crc64.New(table)
}
return r
}
func newFilter64(m, k uint64) *filter64 {
return &filter64{
m: m,
k: k,
h: fnv.New64(),
oh: crc64.New(crc64.MakeTable(crc64.ECMA)),
}
}
func emptyHashes() []HashSum {
return []HashSum{
{Name: "md5", hash: md5.New()},
{Name: "sha1", hash: sha1.New()},
{Name: "sha256", hash: sha256.New()},
{Name: "sha512", hash: sha512.New()},
{Name: "adler32", hash: adler32.New()},
{Name: "crc32 (IEEE)", hash: crc32.New(crc32.MakeTable(crc32.IEEE))},
{Name: "crc32 (Castagnoli)", hash: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{Name: "crc32 (Koopman)", hash: crc32.New(crc32.MakeTable(crc32.Koopman))},
{Name: "crc64 (ISO)", hash: crc64.New(crc64.MakeTable(crc64.ISO))},
{Name: "crc64 (ECMA)", hash: crc64.New(crc64.MakeTable(crc64.ECMA))},
{Name: "fnv32-1", hash: fnv.New32()},
{Name: "fnv32-1a", hash: fnv.New32a()},
{Name: "fnv64-1", hash: fnv.New64()},
{Name: "fnv64-1a", hash: fnv.New64a()},
}
}
// WriteOutputTarStream writes assembled tar archive to a writer.
func WriteOutputTarStream(fg storage.FileGetter, up storage.Unpacker, w io.Writer) error {
// ... Since these are interfaces, this is possible, so let's not have a nil pointer
if fg == nil || up == nil {
return nil
}
var copyBuffer []byte
var crcHash hash.Hash
var crcSum []byte
var multiWriter io.Writer
for {
entry, err := up.Next()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
switch entry.Type {
case storage.SegmentType:
if _, err := w.Write(entry.Payload); err != nil {
return err
}
case storage.FileType:
if entry.Size == 0 {
continue
}
fh, err := fg.Get(entry.GetName())
if err != nil {
return err
}
if crcHash == nil {
crcHash = crc64.New(storage.CRCTable)
crcSum = make([]byte, 8)
multiWriter = io.MultiWriter(w, crcHash)
copyBuffer = byteBufferPool.Get().([]byte)
defer byteBufferPool.Put(copyBuffer)
} else {
crcHash.Reset()
}
if _, err := copyWithBuffer(multiWriter, fh, copyBuffer); err != nil {
fh.Close()
return err
}
if !bytes.Equal(crcHash.Sum(crcSum[:0]), entry.Payload) {
// I would rather this be a comparable ErrInvalidChecksum or such,
// but since it's coming through the PipeReader, the context of
// _which_ file would be lost...
fh.Close()
return fmt.Errorf("file integrity checksum failed for %q", entry.GetName())
}
fh.Close()
}
}
}
func BenchmarkCrc64Hash(b *testing.B) {
const bytes = 1024
s := newsHash(crc64.New(crc64.MakeTable(crc64.ISO)))
dat := make([]byte, bytes)
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Hash(dat)
b.SetBytes(bytes)
}
}
func NewWriter(out io.Writer, enableCRC bool) *Writer {
w := new(Writer)
if enableCRC {
w.crc = crc64.New(table)
w.out = io.MultiWriter(w.crc, out)
} else {
w.out = out
}
return w
}
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
c := crc64.New(CRCTable)
tRdr := io.TeeReader(r, c)
b := bytes.NewBuffer([]byte{})
i, err := io.Copy(b, tRdr)
if err != nil {
return 0, nil, err
}
bfgp.files[name] = b.Bytes()
return i, c.Sum(nil), nil
}
// Build a key set of the keys + a crc64 of the keys (which we can
// efficiently compare). Also returns an index of string to position
func NewKeySet(keys []string, base float64) *KeySet {
c := crc.New(crc.MakeTable(crc.ISO))
index := make(map[string]int)
for idx, s := range keys {
index[s] = idx
c.Write([]byte(s))
}
return internKeySet(&KeySet{Hash: c.Sum64(), Keys: keys, Positions: index, Base: base})
}
func computeHash(path string) uint64 {
f, err := os.Open(path)
if err != nil {
log.Printf("Error in crc: %v", err)
return 0
}
defer f.Close()
h := crc64.New(crcTable)
io.Copy(h, f)
return h.Sum64()
}
func crc64FromValue(sign byte, vv ...interface{}) uint64 {
sub := crc64.New(crcTable)
enc := gob.NewEncoder(sub)
for _, v := range vv {
enc.Encode(v)
}
hash := NewCRC64(sign, 1)
hash.Add(sub.Sum64())
return hash.Sum64()
}
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
crc := crc64.New(CRCTable)
buf := bytes.NewBuffer(nil)
cw := io.MultiWriter(crc, buf)
i, err := io.Copy(cw, r)
if err != nil {
return 0, nil, err
}
bfgp.files[name] = buf.Bytes()
return i, crc.Sum(nil), nil
}
// Create is used to start a new snapshot
func (f *FileSnapshotStore) Create(index, term uint64, peers []byte) (SnapshotSink, error) {
// Create a new path
name := snapshotName(term, index)
path := filepath.Join(f.path, name+tmpSuffix)
f.logger.Printf("[INFO] snapshot: Creating new snapshot at %s", path)
// Make the directory
if err := os.MkdirAll(path, 0755); err != nil {
f.logger.Printf("[ERR] snapshot: Failed to make snapshot directory: %v", err)
return nil, err
}
// Create the sink
sink := &FileSnapshotSink{
store: f,
logger: f.logger,
dir: path,
meta: fileSnapshotMeta{
SnapshotMeta: SnapshotMeta{
ID: name,
Index: index,
Term: term,
Peers: peers,
},
CRC: nil,
},
}
// Write out the meta data
if err := sink.writeMeta(); err != nil {
f.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
return nil, err
}
// Open the state file
statePath := filepath.Join(path, stateFilePath)
fh, err := os.Create(statePath)
if err != nil {
f.logger.Printf("[ERR] snapshot: Failed to create state file: %v", err)
return nil, err
}
sink.stateFile = fh
// Create a CRC64 hash
sink.stateHash = crc64.New(crc64.MakeTable(crc64.ECMA))
// Wrap both the hash and file in a MultiWriter with buffering
multi := io.MultiWriter(sink.stateFile, sink.stateHash)
sink.buffered = bufio.NewWriter(multi)
// Done
return sink, nil
}
// Sum64 calculates a numeric hash sum
func (h *CRC64) Sum64() uint64 {
sort.Sort(h.factors)
hash := crc64.New(crcTable)
hash.Write([]byte{h.sign})
for _, factor := range h.factors {
bin := make([]byte, 8)
binary.LittleEndian.PutUint64(bin, factor)
hash.Write(bin)
}
return hash.Sum64()
}
