Пример #1
2
func GetHash(a string) (hash.Hash, error) {
	var h hash.Hash
	switch a {
	case "adler32":
		h = adler32.New()
	case "crc32", "crc32ieee":
		h = crc32.New(crc32.MakeTable(crc32.IEEE))
	case "crc32castagnoli":
		h = crc32.New(crc32.MakeTable(crc32.Castagnoli))
	case "crc32koopman":
		h = crc32.New(crc32.MakeTable(crc32.Koopman))
	case "crc64", "crc64iso":
		h = crc64.New(crc64.MakeTable(crc64.ISO))
	case "crc64ecma":
		h = crc64.New(crc64.MakeTable(crc64.ECMA))
	case "fnv", "fnv32":
		h = fnv.New32()
	case "fnv32a":
		h = fnv.New32a()
	case "fnv64":
		h = fnv.New64()
	case "fnv64a":
		h = fnv.New64a()
	case "hmac", "hmacsha256":
		h = hmac.New(sha256.New, []byte(key))
	case "hmacmd5":
		h = hmac.New(md5.New, []byte(key))
	case "hmacsha1":
		h = hmac.New(sha1.New, []byte(key))
	case "hmacsha512":
		h = hmac.New(sha512.New, []byte(key))
	case "md4":
		h = md4.New()
	case "md5":
		h = md5.New()
	case "ripemd160":
		h = ripemd160.New()
	case "sha1":
		h = sha1.New()
	case "sha224":
		h = sha256.New224()
	case "sha256":
		h = sha256.New()
	case "sha384":
		h = sha512.New384()
	case "sha512":
		h = sha512.New()
	default:
		return nil, errors.New("Invalid algorithm")
	}
	return h, nil
}
Пример #2
0
func makeHash(name string) hash.Hash {
	switch strings.ToLower(name) {
	case "ripemd160":
		return ripemd160.New()
	case "md4":
		return md4.New()
	case "md5":
		return md5.New()
	case "sha1":
		return sha1.New()
	case "sha256":
		return sha256.New()
	case "sha384":
		return sha512.New384()
	case "sha3-224":
		return sha3.New224()
	case "sha3-256":
		return sha3.New256()
	case "sha3-384":
		return sha3.New384()
	case "sha3-512":
		return sha3.New512()
	case "sha512":
		return sha512.New()
	case "sha512-224":
		return sha512.New512_224()
	case "sha512-256":
		return sha512.New512_256()
	case "crc32-ieee":
		return crc32.NewIEEE()
	case "crc64-iso":
		return crc64.New(crc64.MakeTable(crc64.ISO))
	case "crc64-ecma":
		return crc64.New(crc64.MakeTable(crc64.ECMA))
	case "adler32":
		return adler32.New()
	case "fnv32":
		return fnv.New32()
	case "fnv32a":
		return fnv.New32a()
	case "fnv64":
		return fnv.New64()
	case "fnv64a":
		return fnv.New64a()
	case "xor8":
		return new(xor8)
	case "fletch16":
		return &fletch16{}
	}
	return nil
}
Пример #3
0
func NewSpecialCollectionCache(intent *intents.Intent, demux *Demultiplexer) *SpecialCollectionCache {
	return &SpecialCollectionCache{
		Intent: intent,
		Demux:  demux,
		hash:   crc64.New(crc64.MakeTable(crc64.ECMA)),
	}
}
Пример #4
0
func hashcrc64(s []byte) []byte {
	var tab = crc64.MakeTable(crc64.ECMA)
	h := crc64.New(tab)
	h.Write(s)

	return h.Sum(nil)
}
Пример #5
0
func main() {
	var ECMATable = crc64.MakeTable(crc64.ECMA)
	h := crc64.New(ECMATable)
	h.Write([]byte("test"))
	v := h.Sum64()
	fmt.Println(v)
}
Пример #6
0
func (v *VagrantShareRemoteWatcher) loop(ctx context.Context) {

	table := crc64.MakeTable(crc64.ECMA)
	var lastCheckSum uint64 = 0

	for {

		go func() {
			checkSum, descriptor, err := v.downloadJson(table)

			if err != nil {
				v.OnError <- err
			} else {

				if lastCheckSum != checkSum {

					v.Updated <- descriptor
					lastCheckSum = checkSum
				}
			}
		}()

		select {

		case <-ctx.Done():
			return

		case <-v.clock.After(v.periodInSeconds):
			// continue
		}
	}
}
Пример #7
0
func process_file(filename string, complete chan Sumlist) {
	sumlist := Sumlist{}
	sumlist.filename = filename

	// Open the file and bail if we fail
	infile, err := os.Open(filename)
	if err != nil {
		log.Printf("Unable to open %s: %s", filename, err)
		complete <- sumlist
		return
	}
	defer infile.Close()

	// Create the checksum objects
	if flag_crc32 {
		sumlist.sums = append(sumlist.sums, Checksum{"CRC32", crc32.New(crc32.IEEETable)})
	}
	if flag_crc64 {
		sumlist.sums = append(sumlist.sums, Checksum{"CRC64", crc64.New(crc64.MakeTable(crc64.ISO))})
	}
	if flag_sha224 {
		sumlist.sums = append(sumlist.sums, Checksum{"SHA224", sha256.New224()})
	}
	if flag_sha256 {
		sumlist.sums = append(sumlist.sums, Checksum{"SHA256", sha256.New()})
	}
	if flag_sha384 {
		sumlist.sums = append(sumlist.sums, Checksum{"SHA384", sha512.New384()})
	}
	if flag_sha512 {
		sumlist.sums = append(sumlist.sums, Checksum{"SHA512", sha512.New()})
	}

	// Create our file reader
	reader := bufio.NewReader(infile)

	// Start a buffer and loop to read the entire file
	buf := make([]byte, 4096)
	for {
		read_count, err := reader.Read(buf)
		// If we get an error that is not EOF, then we have a problem
		if err != nil && err != io.EOF {
			log.Printf("Unable to open %s: %s", filename, err)
			complete <- sumlist
			return
		}
		// If the returned size is zero, we're at the end of the file
		if read_count == 0 {
			break
		}

		// Add the buffer contents to the checksum calculation
		for _, sum := range sumlist.sums {
			sum.hashFunc.Write(buf[:read_count])
		}

	}

	complete <- sumlist
}
Пример #8
0
func (e *Engine) crc64_ecma() error {
	data, err := computeHash(crc64.New(crc64.MakeTable(crc64.ECMA)), e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}
Пример #9
0
func main() {
	m := map[string]hash.Hash{
		"had":   adler32.New(),
		"hc32":  crc32.NewIEEE(),
		"hc64e": crc64.New(crc64.MakeTable(crc64.ECMA)),
		"hc64i": crc64.New(crc64.MakeTable(crc64.ISO)),
		"hf32":  fnv.New32(),
		"hf32a": fnv.New32a(),
		"hf64":  fnv.New64(),
		"hf64a": fnv.New64a(),
	}
	for n, h := range m {
		runtime.GC()
		testHash(n, h)
	}
}
Пример #10
0
func (a *Archiver) archiveWriter() error {
	hash := crc64.New(crc64.MakeTable(crc64.ECMA))
	output := io.MultiWriter(a.output, hash)
	blockCount := 0

	_, err := output.Write(fastArchiverHeader)
	if err != nil {
		return err
	}

	for block := range a.blockQueue {
		err = block.writeBlock(output)

		blockCount += 1
		if err == nil && (blockCount%1000) == 0 {
			err = writeChecksumBlock(hash, output)
		}

		if err != nil {
			return err
		}
	}

	return writeChecksumBlock(hash, output)
}
Пример #11
0
func TestUseCrc(t *testing.T) {
	table := crc64.MakeTable(crc64.ISO)
	h := crc64.New(table)

	input := "someUniqueId"
	io.WriteString(h, input)
	sum1 := h.Sum64()

	h.Reset()
	input = "someOtherId"
	io.WriteString(h, input)
	sum2 := h.Sum64()

	if sum1 == sum2 {
		t.Errorf("Sums shouldn't match [%x] [%x]\n", sum1, sum2)
		t.Fail()
		return
	}

	h.Reset()
	input = "someUniqueId"
	io.WriteString(h, input)
	sum3 := h.Sum64()

	if sum1 != sum3 {
		t.Errorf("Sums should match [%x] [%x]\n", sum1, sum3)
		t.Fail()
		return
	}
}
Пример #12
0
func (h *HashTable) hash(k []byte) uint64 {
	hash := crc64.New(crc64.MakeTable(crc64.ISO))

	_, _ = hash.Write(k)

	return hash.Sum64() % uint64(len(h.data))
}
Пример #13
0
func init() {
	// Create the hashing function.
	hasher := crc64.New(crc64.MakeTable(crc64.ECMA))
	h := func(s string) uint64 {
		hasher.Reset()
		hasher.Write([]byte(s))
		return hasher.Sum64()
	}

	// Get the current user name.
	osU, err := user.Current()
	u := "UNKNOW"
	if err == nil {
		u = osU.Username
	}

	// Create the constant to make build a unique ID.
	start := uint64(time.Now().UnixNano())
	user := h(u)
	pid := uint64(os.Getpid())
	// Initialize the channel and blank node type.
	nextVal, tBlank = make(chan string, chanSize), Type("/_")
	enc := base64.StdEncoding
	go func() {
		cnt := uint64(0)
		for {
			bs := []byte(fmt.Sprintf("%x:%x:%x:%x", start, user, pid, cnt))
			nextVal <- enc.EncodeToString(bs)
			cnt++
		}
	}()
}
Пример #14
0
func file(fn string) {
	ext := path.Ext(fn)
	if ext == ".go" {
		return
	}
	in, err := ioutil.ReadFile(fn)
	if err != nil {
		panic(err)
	}
	out, err := os.Create(fn + ".go")
	if err != nil {
		panic(err)
	}
	defer out.Close()

	fmt.Fprintf(out, `package resource

func init() {
	Resource[%q] = []byte{`, filepath.Base(fn))

	for i, b := range in {
		if i == 0 {
			fmt.Fprintf(out, `%d`, b)
		} else {
			fmt.Fprintf(out, `, %d`, b)
		}
	}
	hash := crc64.New(crc64.MakeTable(crc64.ISO))
	hash.Write(in)
	fmt.Fprintf(out, "}\n\tHash[%q] = \"W/\\\"%d\\\"\"\n}\n", filepath.Base(fn), hash.Sum64())
}
Пример #15
0
func newFilter64(m, k uint64) *filter64 {
	return &filter64{
		m:  m,
		k:  k,
		h:  fnv.New64(),
		oh: crc64.New(crc64.MakeTable(crc64.ECMA)),
	}
}
Пример #16
0
func emptyHashes() []HashSum {
	return []HashSum{
		{Name: "md5", hash: md5.New()},
		{Name: "sha1", hash: sha1.New()},
		{Name: "sha256", hash: sha256.New()},
		{Name: "sha512", hash: sha512.New()},
		{Name: "adler32", hash: adler32.New()},
		{Name: "crc32 (IEEE)", hash: crc32.New(crc32.MakeTable(crc32.IEEE))},
		{Name: "crc32 (Castagnoli)", hash: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
		{Name: "crc32 (Koopman)", hash: crc32.New(crc32.MakeTable(crc32.Koopman))},
		{Name: "crc64 (ISO)", hash: crc64.New(crc64.MakeTable(crc64.ISO))},
		{Name: "crc64 (ECMA)", hash: crc64.New(crc64.MakeTable(crc64.ECMA))},
		{Name: "fnv32-1", hash: fnv.New32()},
		{Name: "fnv32-1a", hash: fnv.New32a()},
		{Name: "fnv64-1", hash: fnv.New64()},
		{Name: "fnv64-1a", hash: fnv.New64a()},
	}
}
Пример #17
0
// HashVbuuid return crc64 value of list of 64-bit vbuuids.
func HashVbuuid(vbuuids []uint64) uint64 {
	var bytes []byte
	vbuuids_sl := (*reflect.SliceHeader)(unsafe.Pointer(&vbuuids))
	bytes_sl := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
	bytes_sl.Data = vbuuids_sl.Data
	bytes_sl.Len = vbuuids_sl.Len * 8
	bytes_sl.Cap = vbuuids_sl.Cap * 8
	return crc64.Checksum(bytes, crc64.MakeTable(crc64.ECMA))
}
Пример #18
0
func BenchmarkCrc64Hash(b *testing.B) {
	const bytes = 1024
	s := newsHash(crc64.New(crc64.MakeTable(crc64.ISO)))
	dat := make([]byte, bytes)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		s.Hash(dat)
		b.SetBytes(bytes)
	}
}
Пример #19
0
func FileCRC64(filenameOrURL string) (uint64, error) {
	data, err := FileGetBytes(filenameOrURL)
	if err != nil {
		return 0, err
	}
	if crc64Table == nil {
		crc64Table = crc64.MakeTable(crc64.ECMA)
	}
	return crc64.Checksum(data, crc64Table), nil
}
Пример #20
0
// Build a key set of the keys + a crc64 of the keys (which we can
// efficiently compare). Also returns an index of string to position
func NewKeySet(keys []string, base float64) *KeySet {
	c := crc.New(crc.MakeTable(crc.ISO))
	index := make(map[string]int)

	for idx, s := range keys {
		index[s] = idx
		c.Write([]byte(s))
	}

	return internKeySet(&KeySet{Hash: c.Sum64(), Keys: keys, Positions: index, Base: base})
}
Пример #21
0
// compute a hashcode for ExecutorInfo that may be used as a reasonable litmus test
// with respect to compatibility across HA schedulers. the intent is that an HA scheduler
// should fail-fast if it doesn't pass this test, rather than generating (potentially many)
// errors at run-time because a Mesos master decides that the ExecutorInfo generated by a
// secondary scheduler doesn't match that of the primary scheduler.
//
// Note: We intentionally leave out the Resources in this hash because they are
//       set during procurement and should not lead to a different ExecutorId.
//       This also means that the Resources do not contribute to offer
//       compatibility checking. But as we persist and restore the Resources
//       through node anotation we make sure that the right resources are chosen
//       during task launch.
//
// see https://github.com/apache/mesos/blob/0.22.0/src/common/type_utils.cpp#L110
func hash(info *mesos.ExecutorInfo) uint64 {
	// !!! we specifically do NOT include:
	// - Framework ID because it's a value that's initialized too late for us to use
	// - Executor ID because it's a value that includes a copy of this hash
	buf := &bytes.Buffer{}
	buf.WriteString(info.GetName())
	buf.WriteString(info.GetSource())
	buf.Write(info.Data)

	if info.Command != nil {
		buf.WriteString(info.Command.GetValue())
		buf.WriteString(info.Command.GetUser())
		buf.WriteString(strconv.FormatBool(info.Command.GetShell()))
		if sz := len(info.Command.Arguments); sz > 0 {
			x := make([]string, sz)
			copy(x, info.Command.Arguments)
			sort.Strings(x)
			for _, item := range x {
				buf.WriteString(item)
			}
		}
		if vars := info.Command.Environment.GetVariables(); len(vars) > 0 {
			names := []string{}
			e := make(map[string]string)

			for _, v := range vars {
				if name := v.GetName(); name != "" {
					names = append(names, name)
					e[name] = v.GetValue()
				}
			}
			sort.Strings(names)
			for _, n := range names {
				buf.WriteString(n)
				buf.WriteString("=")
				buf.WriteString(e[n])
			}
		}
		if uris := info.Command.GetUris(); len(uris) > 0 {
			su := []string{}
			for _, uri := range uris {
				su = append(su, fmt.Sprintf("%s%t%t", uri.GetValue(), uri.GetExecutable(), uri.GetExtract()))
			}
			sort.Strings(su)
			for _, uri := range su {
				buf.WriteString(uri)
			}
		}
		//TODO(jdef) add support for Container
	}
	table := crc64.MakeTable(crc64.ECMA)
	return crc64.Checksum(buf.Bytes(), table)
}
Пример #22
0
// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
	var seedInt int64
	if seed != "" {
		crcTable := crc64.MakeTable(crc64.ISO)
		seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
	} else {
		seedInt = time.Now().Unix()
	}

	randSource := rand.NewSource(seedInt)
	return rand.New(randSource)
}
Пример #23
0
// Create is used to start a new snapshot
func (f *FileSnapshotStore) Create(index, term uint64, peers []byte) (SnapshotSink, error) {
	// Create a new path
	name := snapshotName(term, index)
	path := filepath.Join(f.path, name+tmpSuffix)
	f.logger.Printf("[INFO] snapshot: Creating new snapshot at %s", path)

	// Make the directory
	if err := os.MkdirAll(path, 0755); err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to make snapshot directory: %v", err)
		return nil, err
	}

	// Create the sink
	sink := &FileSnapshotSink{
		store:  f,
		logger: f.logger,
		dir:    path,
		meta: fileSnapshotMeta{
			SnapshotMeta: SnapshotMeta{
				ID:    name,
				Index: index,
				Term:  term,
				Peers: peers,
			},
			CRC: nil,
		},
	}

	// Write out the meta data
	if err := sink.writeMeta(); err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
		return nil, err
	}

	// Open the state file
	statePath := filepath.Join(path, stateFilePath)
	fh, err := os.Create(statePath)
	if err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to create state file: %v", err)
		return nil, err
	}
	sink.stateFile = fh

	// Create a CRC64 hash
	sink.stateHash = crc64.New(crc64.MakeTable(crc64.ECMA))

	// Wrap both the hash and file in a MultiWriter with buffering
	multi := io.MultiWriter(sink.stateFile, sink.stateHash)
	sink.buffered = bufio.NewWriter(multi)

	// Done
	return sink, nil
}
Пример #24
0
// WorBig generates string for multi-precision integer n, count of words
// calculated as:
//     (bitLength(n) / 16) rounded towards infinity
func WorBig(n *big.Int) string {
	switch {
	case n.BitLen() <= 16:
		return Wor(n.Int64(), 1)
	case n.BitLen() <= 32:
		return Wor(n.Int64(), 2)
	case n.BitLen() <= 64:
		return Wor(n.Int64(), 4)
	default:
		sum := crc64.Checksum(n.Bytes(), crc64.MakeTable(crc64.ECMA))
		return Wor(int64(sum), (n.BitLen()+16)/16)
	}
}
Пример #25
0
// Open is implemented in Mux.open, but in short, it creates chans and a select case
// and adds the SelectCase and the MuxIn in to the Multiplexer.
func (muxIn *MuxIn) Open() error {
	log.Logf(log.DebugHigh, "MuxIn open %v", muxIn.Intent.Namespace())
	muxIn.writeChan = make(chan []byte)
	muxIn.writeLenChan = make(chan int)
	muxIn.writeCloseFinishedChan = make(chan struct{})
	muxIn.buf = make([]byte, 0, bufferSize)
	muxIn.hash = crc64.New(crc64.MakeTable(crc64.ECMA))
	if bufferWrites {
		muxIn.buf = make([]byte, 0, db.MaxBSONSize)
	}
	muxIn.Mux.Control <- muxIn
	return nil
}
Пример #26
0
func Benchmark_Crc64Block(b *testing.B) {
	b.StopTimer()

	buff := makeBlock(256 * 1024)
	hash := crc64.New(crc64.MakeTable(crc64.ECMA))

	b.StartTimer()
	for ii := 0; ii < b.N; ii++ {
		hash.Write(buff)
		hash.Sum64()
		hash.Reset()
	}
}
Пример #27
0
// Open is part of the intents.file interface.  It creates the chan's in the
// RegularCollectionReceiver and adds the RegularCollectionReceiver to the set of
// RegularCollectonReceivers in the demultiplexer
func (receiver *RegularCollectionReceiver) Open() error {
	// TODO move this implementation to some non intents.file method, to be called from prioritizer.Get
	// So that we don't have to enable this double open stuff.
	// Currently the open needs to finish before the prioritizer.Get finishes, so we open the intents.file
	// in prioritizer.Get even though it's going to get opened again in DumpIntent.
	receiver.openOnce.Do(func() {
		receiver.readLenChan = make(chan int)
		receiver.readBufChan = make(chan []byte)
		receiver.hash = crc64.New(crc64.MakeTable(crc64.ECMA))
		receiver.Demux.Open(receiver.Origin, receiver)
	})
	return nil
}
Пример #28
0
// Open takes a snapshot ID and returns a ReadCloser for that snapshot.
func (f *FileSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
	// Get the metadata
	meta, err := f.readMeta(id)
	if err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to get meta data to open snapshot: %v", err)
		return nil, nil, err
	}

	// Open the state file
	statePath := filepath.Join(f.path, id, stateFilePath)
	fh, err := os.Open(statePath)
	if err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to open state file: %v", err)
		return nil, nil, err
	}

	// Create a CRC64 hash
	stateHash := crc64.New(crc64.MakeTable(crc64.ECMA))

	// Compute the hash
	_, err = io.Copy(stateHash, fh)
	if err != nil {
		f.logger.Printf("[ERR] snapshot: Failed to read state file: %v", err)
		fh.Close()
		return nil, nil, err
	}

	// Verify the hash
	computed := stateHash.Sum(nil)
	if bytes.Compare(meta.CRC, computed) != 0 {
		f.logger.Printf("[ERR] snapshot: CRC checksum failed (stored: %v computed: %v)",
			meta.CRC, computed)
		fh.Close()
		return nil, nil, fmt.Errorf("CRC mismatch")
	}

	// Seek to the start
	if _, err := fh.Seek(0, 0); err != nil {
		f.logger.Printf("[ERR] snapshot: State file seek failed: %v", err)
		fh.Close()
		return nil, nil, err
	}

	// Return a buffered file
	buffered := &bufferedFile{
		bh: bufio.NewReader(fh),
		fh: fh,
	}

	return &meta.SnapshotMeta, buffered, nil
}
Пример #29
0
func TestBloomFilter(t *testing.T) {
	l := []uint{uint(len(web2)), 200000, 100000, 50000}
	h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
	n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}

	for i := range l {
		for j := range h {
			fmt.Printf("\n\nTesting %s with size %d\n", n[j], l[i])
			bf := New(l[i])
			bf.SetHasher(h[j])
			testBloomFilter(t, bf)
		}
	}
}
Пример #30
0
// Open installs the DemuxOut as the handler for data for the namespace ns
func (demux *Demultiplexer) Open(ns string, out DemuxOut) {
	// In the current implementation where this is either called before the demultiplexing is running
	// or while the demutiplexer is inside of the NamespaceChan NamespaceErrorChan conversation
	// I think that we don't need to lock outs, but I suspect that if the implementation changes
	// we may need to lock when outs is accessed
	log.Logf(log.DebugHigh, "demux Open")
	if demux.outs == nil {
		demux.outs = make(map[string]DemuxOut)
		demux.hashes = make(map[string]hash.Hash64)
		demux.lengths = make(map[string]int64)
	}
	demux.outs[ns] = out
	demux.hashes[ns] = crc64.New(crc64.MakeTable(crc64.ECMA))
	demux.lengths[ns] = 0
}