func GetHash(a string) (hash.Hash, error) {
var h hash.Hash
switch a {
case "adler32":
h = adler32.New()
case "crc32", "crc32ieee":
h = crc32.New(crc32.MakeTable(crc32.IEEE))
case "crc32castagnoli":
h = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case "crc32koopman":
h = crc32.New(crc32.MakeTable(crc32.Koopman))
case "crc64", "crc64iso":
h = crc64.New(crc64.MakeTable(crc64.ISO))
case "crc64ecma":
h = crc64.New(crc64.MakeTable(crc64.ECMA))
case "fnv", "fnv32":
h = fnv.New32()
case "fnv32a":
h = fnv.New32a()
case "fnv64":
h = fnv.New64()
case "fnv64a":
h = fnv.New64a()
case "hmac", "hmacsha256":
h = hmac.New(sha256.New, []byte(key))
case "hmacmd5":
h = hmac.New(md5.New, []byte(key))
case "hmacsha1":
h = hmac.New(sha1.New, []byte(key))
case "hmacsha512":
h = hmac.New(sha512.New, []byte(key))
case "md4":
h = md4.New()
case "md5":
h = md5.New()
case "ripemd160":
h = ripemd160.New()
case "sha1":
h = sha1.New()
case "sha224":
h = sha256.New224()
case "sha256":
h = sha256.New()
case "sha384":
h = sha512.New384()
case "sha512":
h = sha512.New()
default:
return nil, errors.New("Invalid algorithm")
}
return h, nil
}
func makeHash(name string) hash.Hash {
switch strings.ToLower(name) {
case "ripemd160":
return ripemd160.New()
case "md4":
return md4.New()
case "md5":
return md5.New()
case "sha1":
return sha1.New()
case "sha256":
return sha256.New()
case "sha384":
return sha512.New384()
case "sha3-224":
return sha3.New224()
case "sha3-256":
return sha3.New256()
case "sha3-384":
return sha3.New384()
case "sha3-512":
return sha3.New512()
case "sha512":
return sha512.New()
case "sha512-224":
return sha512.New512_224()
case "sha512-256":
return sha512.New512_256()
case "crc32-ieee":
return crc32.NewIEEE()
case "crc64-iso":
return crc64.New(crc64.MakeTable(crc64.ISO))
case "crc64-ecma":
return crc64.New(crc64.MakeTable(crc64.ECMA))
case "adler32":
return adler32.New()
case "fnv32":
return fnv.New32()
case "fnv32a":
return fnv.New32a()
case "fnv64":
return fnv.New64()
case "fnv64a":
return fnv.New64a()
case "xor8":
return new(xor8)
case "fletch16":
return &fletch16{}
}
return nil
}
func NewSpecialCollectionCache(intent *intents.Intent, demux *Demultiplexer) *SpecialCollectionCache {
return &SpecialCollectionCache{
Intent: intent,
Demux: demux,
hash: crc64.New(crc64.MakeTable(crc64.ECMA)),
}
}
func hashcrc64(s []byte) []byte {
var tab = crc64.MakeTable(crc64.ECMA)
h := crc64.New(tab)
h.Write(s)
return h.Sum(nil)
}
func main() {
var ECMATable = crc64.MakeTable(crc64.ECMA)
h := crc64.New(ECMATable)
h.Write([]byte("test"))
v := h.Sum64()
fmt.Println(v)
}
func (v *VagrantShareRemoteWatcher) loop(ctx context.Context) {
table := crc64.MakeTable(crc64.ECMA)
var lastCheckSum uint64 = 0
for {
go func() {
checkSum, descriptor, err := v.downloadJson(table)
if err != nil {
v.OnError <- err
} else {
if lastCheckSum != checkSum {
v.Updated <- descriptor
lastCheckSum = checkSum
}
}
}()
select {
case <-ctx.Done():
return
case <-v.clock.After(v.periodInSeconds):
// continue
}
}
}
func process_file(filename string, complete chan Sumlist) {
sumlist := Sumlist{}
sumlist.filename = filename
// Open the file and bail if we fail
infile, err := os.Open(filename)
if err != nil {
log.Printf("Unable to open %s: %s", filename, err)
complete <- sumlist
return
}
defer infile.Close()
// Create the checksum objects
if flag_crc32 {
sumlist.sums = append(sumlist.sums, Checksum{"CRC32", crc32.New(crc32.IEEETable)})
}
if flag_crc64 {
sumlist.sums = append(sumlist.sums, Checksum{"CRC64", crc64.New(crc64.MakeTable(crc64.ISO))})
}
if flag_sha224 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA224", sha256.New224()})
}
if flag_sha256 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA256", sha256.New()})
}
if flag_sha384 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA384", sha512.New384()})
}
if flag_sha512 {
sumlist.sums = append(sumlist.sums, Checksum{"SHA512", sha512.New()})
}
// Create our file reader
reader := bufio.NewReader(infile)
// Start a buffer and loop to read the entire file
buf := make([]byte, 4096)
for {
read_count, err := reader.Read(buf)
// If we get an error that is not EOF, then we have a problem
if err != nil && err != io.EOF {
log.Printf("Unable to open %s: %s", filename, err)
complete <- sumlist
return
}
// If the returned size is zero, we're at the end of the file
if read_count == 0 {
break
}
// Add the buffer contents to the checksum calculation
for _, sum := range sumlist.sums {
sum.hashFunc.Write(buf[:read_count])
}
}
complete <- sumlist
}
func (e *Engine) crc64_ecma() error {
data, err := computeHash(crc64.New(crc64.MakeTable(crc64.ECMA)), e.stack.Pop())
if err == nil {
e.stack.Push(data)
}
return err
}
func main() {
m := map[string]hash.Hash{
"had": adler32.New(),
"hc32": crc32.NewIEEE(),
"hc64e": crc64.New(crc64.MakeTable(crc64.ECMA)),
"hc64i": crc64.New(crc64.MakeTable(crc64.ISO)),
"hf32": fnv.New32(),
"hf32a": fnv.New32a(),
"hf64": fnv.New64(),
"hf64a": fnv.New64a(),
}
for n, h := range m {
runtime.GC()
testHash(n, h)
}
}
func (a *Archiver) archiveWriter() error {
hash := crc64.New(crc64.MakeTable(crc64.ECMA))
output := io.MultiWriter(a.output, hash)
blockCount := 0
_, err := output.Write(fastArchiverHeader)
if err != nil {
return err
}
for block := range a.blockQueue {
err = block.writeBlock(output)
blockCount += 1
if err == nil && (blockCount%1000) == 0 {
err = writeChecksumBlock(hash, output)
}
if err != nil {
return err
}
}
return writeChecksumBlock(hash, output)
}
func TestUseCrc(t *testing.T) {
table := crc64.MakeTable(crc64.ISO)
h := crc64.New(table)
input := "someUniqueId"
io.WriteString(h, input)
sum1 := h.Sum64()
h.Reset()
input = "someOtherId"
io.WriteString(h, input)
sum2 := h.Sum64()
if sum1 == sum2 {
t.Errorf("Sums shouldn't match [%x] [%x]\n", sum1, sum2)
t.Fail()
return
}
h.Reset()
input = "someUniqueId"
io.WriteString(h, input)
sum3 := h.Sum64()
if sum1 != sum3 {
t.Errorf("Sums should match [%x] [%x]\n", sum1, sum3)
t.Fail()
return
}
}
func (h *HashTable) hash(k []byte) uint64 {
hash := crc64.New(crc64.MakeTable(crc64.ISO))
_, _ = hash.Write(k)
return hash.Sum64() % uint64(len(h.data))
}
func init() {
// Create the hashing function.
hasher := crc64.New(crc64.MakeTable(crc64.ECMA))
h := func(s string) uint64 {
hasher.Reset()
hasher.Write([]byte(s))
return hasher.Sum64()
}
// Get the current user name.
osU, err := user.Current()
u := "UNKNOW"
if err == nil {
u = osU.Username
}
// Create the constant to make build a unique ID.
start := uint64(time.Now().UnixNano())
user := h(u)
pid := uint64(os.Getpid())
// Initialize the channel and blank node type.
nextVal, tBlank = make(chan string, chanSize), Type("/_")
enc := base64.StdEncoding
go func() {
cnt := uint64(0)
for {
bs := []byte(fmt.Sprintf("%x:%x:%x:%x", start, user, pid, cnt))
nextVal <- enc.EncodeToString(bs)
cnt++
}
}()
}
func file(fn string) {
ext := path.Ext(fn)
if ext == ".go" {
return
}
in, err := ioutil.ReadFile(fn)
if err != nil {
panic(err)
}
out, err := os.Create(fn + ".go")
if err != nil {
panic(err)
}
defer out.Close()
fmt.Fprintf(out, `package resource
func init() {
Resource[%q] = []byte{`, filepath.Base(fn))
for i, b := range in {
if i == 0 {
fmt.Fprintf(out, `%d`, b)
} else {
fmt.Fprintf(out, `, %d`, b)
}
}
hash := crc64.New(crc64.MakeTable(crc64.ISO))
hash.Write(in)
fmt.Fprintf(out, "}\n\tHash[%q] = \"W/\\\"%d\\\"\"\n}\n", filepath.Base(fn), hash.Sum64())
}
func newFilter64(m, k uint64) *filter64 {
return &filter64{
m: m,
k: k,
h: fnv.New64(),
oh: crc64.New(crc64.MakeTable(crc64.ECMA)),
}
}
func emptyHashes() []HashSum {
return []HashSum{
{Name: "md5", hash: md5.New()},
{Name: "sha1", hash: sha1.New()},
{Name: "sha256", hash: sha256.New()},
{Name: "sha512", hash: sha512.New()},
{Name: "adler32", hash: adler32.New()},
{Name: "crc32 (IEEE)", hash: crc32.New(crc32.MakeTable(crc32.IEEE))},
{Name: "crc32 (Castagnoli)", hash: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{Name: "crc32 (Koopman)", hash: crc32.New(crc32.MakeTable(crc32.Koopman))},
{Name: "crc64 (ISO)", hash: crc64.New(crc64.MakeTable(crc64.ISO))},
{Name: "crc64 (ECMA)", hash: crc64.New(crc64.MakeTable(crc64.ECMA))},
{Name: "fnv32-1", hash: fnv.New32()},
{Name: "fnv32-1a", hash: fnv.New32a()},
{Name: "fnv64-1", hash: fnv.New64()},
{Name: "fnv64-1a", hash: fnv.New64a()},
}
}
// HashVbuuid return crc64 value of list of 64-bit vbuuids.
func HashVbuuid(vbuuids []uint64) uint64 {
var bytes []byte
vbuuids_sl := (*reflect.SliceHeader)(unsafe.Pointer(&vbuuids))
bytes_sl := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
bytes_sl.Data = vbuuids_sl.Data
bytes_sl.Len = vbuuids_sl.Len * 8
bytes_sl.Cap = vbuuids_sl.Cap * 8
return crc64.Checksum(bytes, crc64.MakeTable(crc64.ECMA))
}
func BenchmarkCrc64Hash(b *testing.B) {
const bytes = 1024
s := newsHash(crc64.New(crc64.MakeTable(crc64.ISO)))
dat := make([]byte, bytes)
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Hash(dat)
b.SetBytes(bytes)
}
}
func FileCRC64(filenameOrURL string) (uint64, error) {
data, err := FileGetBytes(filenameOrURL)
if err != nil {
return 0, err
}
if crc64Table == nil {
crc64Table = crc64.MakeTable(crc64.ECMA)
}
return crc64.Checksum(data, crc64Table), nil
}
// Build a key set of the keys + a crc64 of the keys (which we can
// efficiently compare). Also returns an index of string to position
func NewKeySet(keys []string, base float64) *KeySet {
c := crc.New(crc.MakeTable(crc.ISO))
index := make(map[string]int)
for idx, s := range keys {
index[s] = idx
c.Write([]byte(s))
}
return internKeySet(&KeySet{Hash: c.Sum64(), Keys: keys, Positions: index, Base: base})
}
// compute a hashcode for ExecutorInfo that may be used as a reasonable litmus test
// with respect to compatibility across HA schedulers. the intent is that an HA scheduler
// should fail-fast if it doesn't pass this test, rather than generating (potentially many)
// errors at run-time because a Mesos master decides that the ExecutorInfo generated by a
// secondary scheduler doesn't match that of the primary scheduler.
//
// Note: We intentionally leave out the Resources in this hash because they are
// set during procurement and should not lead to a different ExecutorId.
// This also means that the Resources do not contribute to offer
// compatibility checking. But as we persist and restore the Resources
// through node anotation we make sure that the right resources are chosen
// during task launch.
//
// see https://github.com/apache/mesos/blob/0.22.0/src/common/type_utils.cpp#L110
func hash(info *mesos.ExecutorInfo) uint64 {
// !!! we specifically do NOT include:
// - Framework ID because it's a value that's initialized too late for us to use
// - Executor ID because it's a value that includes a copy of this hash
buf := &bytes.Buffer{}
buf.WriteString(info.GetName())
buf.WriteString(info.GetSource())
buf.Write(info.Data)
if info.Command != nil {
buf.WriteString(info.Command.GetValue())
buf.WriteString(info.Command.GetUser())
buf.WriteString(strconv.FormatBool(info.Command.GetShell()))
if sz := len(info.Command.Arguments); sz > 0 {
x := make([]string, sz)
copy(x, info.Command.Arguments)
sort.Strings(x)
for _, item := range x {
buf.WriteString(item)
}
}
if vars := info.Command.Environment.GetVariables(); len(vars) > 0 {
names := []string{}
e := make(map[string]string)
for _, v := range vars {
if name := v.GetName(); name != "" {
names = append(names, name)
e[name] = v.GetValue()
}
}
sort.Strings(names)
for _, n := range names {
buf.WriteString(n)
buf.WriteString("=")
buf.WriteString(e[n])
}
}
if uris := info.Command.GetUris(); len(uris) > 0 {
su := []string{}
for _, uri := range uris {
su = append(su, fmt.Sprintf("%s%t%t", uri.GetValue(), uri.GetExecutable(), uri.GetExtract()))
}
sort.Strings(su)
for _, uri := range su {
buf.WriteString(uri)
}
}
//TODO(jdef) add support for Container
}
table := crc64.MakeTable(crc64.ECMA)
return crc64.Checksum(buf.Bytes(), table)
}
// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
var seedInt int64
if seed != "" {
crcTable := crc64.MakeTable(crc64.ISO)
seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
} else {
seedInt = time.Now().Unix()
}
randSource := rand.NewSource(seedInt)
return rand.New(randSource)
}
// Create is used to start a new snapshot
func (f *FileSnapshotStore) Create(index, term uint64, peers []byte) (SnapshotSink, error) {
// Create a new path
name := snapshotName(term, index)
path := filepath.Join(f.path, name+tmpSuffix)
f.logger.Printf("[INFO] snapshot: Creating new snapshot at %s", path)
// Make the directory
if err := os.MkdirAll(path, 0755); err != nil {
f.logger.Printf("[ERR] snapshot: Failed to make snapshot directory: %v", err)
return nil, err
}
// Create the sink
sink := &FileSnapshotSink{
store: f,
logger: f.logger,
dir: path,
meta: fileSnapshotMeta{
SnapshotMeta: SnapshotMeta{
ID: name,
Index: index,
Term: term,
Peers: peers,
},
CRC: nil,
},
}
// Write out the meta data
if err := sink.writeMeta(); err != nil {
f.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
return nil, err
}
// Open the state file
statePath := filepath.Join(path, stateFilePath)
fh, err := os.Create(statePath)
if err != nil {
f.logger.Printf("[ERR] snapshot: Failed to create state file: %v", err)
return nil, err
}
sink.stateFile = fh
// Create a CRC64 hash
sink.stateHash = crc64.New(crc64.MakeTable(crc64.ECMA))
// Wrap both the hash and file in a MultiWriter with buffering
multi := io.MultiWriter(sink.stateFile, sink.stateHash)
sink.buffered = bufio.NewWriter(multi)
// Done
return sink, nil
}
// WorBig generates string for multi-precision integer n, count of words
// calculated as:
// (bitLength(n) / 16) rounded towards infinity
func WorBig(n *big.Int) string {
switch {
case n.BitLen() <= 16:
return Wor(n.Int64(), 1)
case n.BitLen() <= 32:
return Wor(n.Int64(), 2)
case n.BitLen() <= 64:
return Wor(n.Int64(), 4)
default:
sum := crc64.Checksum(n.Bytes(), crc64.MakeTable(crc64.ECMA))
return Wor(int64(sum), (n.BitLen()+16)/16)
}
}
// Open is implemented in Mux.open, but in short, it creates chans and a select case
// and adds the SelectCase and the MuxIn in to the Multiplexer.
func (muxIn *MuxIn) Open() error {
log.Logf(log.DebugHigh, "MuxIn open %v", muxIn.Intent.Namespace())
muxIn.writeChan = make(chan []byte)
muxIn.writeLenChan = make(chan int)
muxIn.writeCloseFinishedChan = make(chan struct{})
muxIn.buf = make([]byte, 0, bufferSize)
muxIn.hash = crc64.New(crc64.MakeTable(crc64.ECMA))
if bufferWrites {
muxIn.buf = make([]byte, 0, db.MaxBSONSize)
}
muxIn.Mux.Control <- muxIn
return nil
}
func Benchmark_Crc64Block(b *testing.B) {
b.StopTimer()
buff := makeBlock(256 * 1024)
hash := crc64.New(crc64.MakeTable(crc64.ECMA))
b.StartTimer()
for ii := 0; ii < b.N; ii++ {
hash.Write(buff)
hash.Sum64()
hash.Reset()
}
}
// Open is part of the intents.file interface. It creates the chan's in the
// RegularCollectionReceiver and adds the RegularCollectionReceiver to the set of
// RegularCollectonReceivers in the demultiplexer
func (receiver *RegularCollectionReceiver) Open() error {
// TODO move this implementation to some non intents.file method, to be called from prioritizer.Get
// So that we don't have to enable this double open stuff.
// Currently the open needs to finish before the prioritizer.Get finishes, so we open the intents.file
// in prioritizer.Get even though it's going to get opened again in DumpIntent.
receiver.openOnce.Do(func() {
receiver.readLenChan = make(chan int)
receiver.readBufChan = make(chan []byte)
receiver.hash = crc64.New(crc64.MakeTable(crc64.ECMA))
receiver.Demux.Open(receiver.Origin, receiver)
})
return nil
}
// Open takes a snapshot ID and returns a ReadCloser for that snapshot.
func (f *FileSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
// Get the metadata
meta, err := f.readMeta(id)
if err != nil {
f.logger.Printf("[ERR] snapshot: Failed to get meta data to open snapshot: %v", err)
return nil, nil, err
}
// Open the state file
statePath := filepath.Join(f.path, id, stateFilePath)
fh, err := os.Open(statePath)
if err != nil {
f.logger.Printf("[ERR] snapshot: Failed to open state file: %v", err)
return nil, nil, err
}
// Create a CRC64 hash
stateHash := crc64.New(crc64.MakeTable(crc64.ECMA))
// Compute the hash
_, err = io.Copy(stateHash, fh)
if err != nil {
f.logger.Printf("[ERR] snapshot: Failed to read state file: %v", err)
fh.Close()
return nil, nil, err
}
// Verify the hash
computed := stateHash.Sum(nil)
if bytes.Compare(meta.CRC, computed) != 0 {
f.logger.Printf("[ERR] snapshot: CRC checksum failed (stored: %v computed: %v)",
meta.CRC, computed)
fh.Close()
return nil, nil, fmt.Errorf("CRC mismatch")
}
// Seek to the start
if _, err := fh.Seek(0, 0); err != nil {
f.logger.Printf("[ERR] snapshot: State file seek failed: %v", err)
fh.Close()
return nil, nil, err
}
// Return a buffered file
buffered := &bufferedFile{
bh: bufio.NewReader(fh),
fh: fh,
}
return &meta.SnapshotMeta, buffered, nil
}
func TestBloomFilter(t *testing.T) {
l := []uint{uint(len(web2)), 200000, 100000, 50000}
h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}
for i := range l {
for j := range h {
fmt.Printf("\n\nTesting %s with size %d\n", n[j], l[i])
bf := New(l[i])
bf.SetHasher(h[j])
testBloomFilter(t, bf)
}
}
}
// Open installs the DemuxOut as the handler for data for the namespace ns
func (demux *Demultiplexer) Open(ns string, out DemuxOut) {
// In the current implementation where this is either called before the demultiplexing is running
// or while the demutiplexer is inside of the NamespaceChan NamespaceErrorChan conversation
// I think that we don't need to lock outs, but I suspect that if the implementation changes
// we may need to lock when outs is accessed
log.Logf(log.DebugHigh, "demux Open")
if demux.outs == nil {
demux.outs = make(map[string]DemuxOut)
demux.hashes = make(map[string]hash.Hash64)
demux.lengths = make(map[string]int64)
}
demux.outs[ns] = out
demux.hashes[ns] = crc64.New(crc64.MakeTable(crc64.ECMA))
demux.lengths[ns] = 0
}
