Beispiel #1
0
func (id *Id) Partition(max uint64) uint64 {
	b, err := id.Encode()
	if err != nil {
		panic(err)
	}
	return crc64.Checksum(b, partitionTable) % max
}
Beispiel #2
0
func BenchmarkCRC64ISOShort(b *testing.B) {
	var bv uint64
	k := []byte("Test-key-100")
	for i := 0; i < b.N; i++ {
		bv += crc64.Checksum(k, crc64ISO)
	}
}
Beispiel #3
0
func (m M) Hash() int64 {
	if m.X != nil {
		return m.X.Hash()
	} else if len(m.S) == 0 {
		return m.N
	}

	return int64(crc64.Checksum([]byte(m.S), CrcPolynomial))
}
Beispiel #4
0
// HashVbuuid return crc64 value of list of 64-bit vbuuids.
func HashVbuuid(vbuuids []uint64) uint64 {
	var bytes []byte
	vbuuids_sl := (*reflect.SliceHeader)(unsafe.Pointer(&vbuuids))
	bytes_sl := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
	bytes_sl.Data = vbuuids_sl.Data
	bytes_sl.Len = vbuuids_sl.Len * 8
	bytes_sl.Cap = vbuuids_sl.Cap * 8
	return crc64.Checksum(bytes, crc64.MakeTable(crc64.ECMA))
}
Beispiel #5
0
func FileCRC64(filenameOrURL string) (uint64, error) {
	data, err := FileGetBytes(filenameOrURL)
	if err != nil {
		return 0, err
	}
	if crc64Table == nil {
		crc64Table = crc64.MakeTable(crc64.ECMA)
	}
	return crc64.Checksum(data, crc64Table), nil
}
Beispiel #6
0
func (h *crcHash) Sum64() uint64 {
	sort.Sort(h.factors)

	data := make([]byte, len(h.factors)*8+1)
	data[0] = h.prefix
	for i, factor := range h.factors {
		binary.LittleEndian.PutUint64(data[i*8+1:], factor)
	}
	return crc64.Checksum(data, crcTable)
}
Beispiel #7
0
// compute a hashcode for ExecutorInfo that may be used as a reasonable litmus test
// with respect to compatibility across HA schedulers. the intent is that an HA scheduler
// should fail-fast if it doesn't pass this test, rather than generating (potentially many)
// errors at run-time because a Mesos master decides that the ExecutorInfo generated by a
// secondary scheduler doesn't match that of the primary scheduler.
//
// Note: We intentionally leave out the Resources in this hash because they are
//       set during procurement and should not lead to a different ExecutorId.
//       This also means that the Resources do not contribute to offer
//       compatibility checking. But as we persist and restore the Resources
//       through node anotation we make sure that the right resources are chosen
//       during task launch.
//
// see https://github.com/apache/mesos/blob/0.22.0/src/common/type_utils.cpp#L110
func hash(info *mesos.ExecutorInfo) uint64 {
	// !!! we specifically do NOT include:
	// - Framework ID because it's a value that's initialized too late for us to use
	// - Executor ID because it's a value that includes a copy of this hash
	buf := &bytes.Buffer{}
	buf.WriteString(info.GetName())
	buf.WriteString(info.GetSource())
	buf.Write(info.Data)

	if info.Command != nil {
		buf.WriteString(info.Command.GetValue())
		buf.WriteString(info.Command.GetUser())
		buf.WriteString(strconv.FormatBool(info.Command.GetShell()))
		if sz := len(info.Command.Arguments); sz > 0 {
			x := make([]string, sz)
			copy(x, info.Command.Arguments)
			sort.Strings(x)
			for _, item := range x {
				buf.WriteString(item)
			}
		}
		if vars := info.Command.Environment.GetVariables(); len(vars) > 0 {
			names := []string{}
			e := make(map[string]string)

			for _, v := range vars {
				if name := v.GetName(); name != "" {
					names = append(names, name)
					e[name] = v.GetValue()
				}
			}
			sort.Strings(names)
			for _, n := range names {
				buf.WriteString(n)
				buf.WriteString("=")
				buf.WriteString(e[n])
			}
		}
		if uris := info.Command.GetUris(); len(uris) > 0 {
			su := []string{}
			for _, uri := range uris {
				su = append(su, fmt.Sprintf("%s%t%t", uri.GetValue(), uri.GetExecutable(), uri.GetExtract()))
			}
			sort.Strings(su)
			for _, uri := range su {
				buf.WriteString(uri)
			}
		}
		//TODO(jdef) add support for Container
	}
	table := crc64.MakeTable(crc64.ECMA)
	return crc64.Checksum(buf.Bytes(), table)
}
Beispiel #8
0
// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
	var seedInt int64
	if seed != "" {
		crcTable := crc64.MakeTable(crc64.ISO)
		seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
	} else {
		seedInt = time.Now().Unix()
	}

	randSource := rand.NewSource(seedInt)
	return rand.New(randSource)
}
Beispiel #9
0
// WorBig generates string for multi-precision integer n, count of words
// calculated as:
//     (bitLength(n) / 16) rounded towards infinity
func WorBig(n *big.Int) string {
	switch {
	case n.BitLen() <= 16:
		return Wor(n.Int64(), 1)
	case n.BitLen() <= 32:
		return Wor(n.Int64(), 2)
	case n.BitLen() <= 64:
		return Wor(n.Int64(), 4)
	default:
		sum := crc64.Checksum(n.Bytes(), crc64.MakeTable(crc64.ECMA))
		return Wor(int64(sum), (n.BitLen()+16)/16)
	}
}
Beispiel #10
0
func (v *VagrantShareRemoteWatcher) downloadJson(table *crc64.Table) (uint64, *VagrantBoxDescripter, error) {

	body, err := v.jsonGetter(v.url)

	if err != nil {
		return 0, nil, err
	}
	checksum64 := crc64.Checksum(body, table)

	var descripter VagrantBoxDescripter

	json.Unmarshal(body, &descripter)

	return checksum64, &descripter, nil
}
Beispiel #11
0
func calculateCRC64Hash(filename string) string {
	var buffer []byte = make([]byte, 1024**NumberOfBytesToRead)

	f, err := os.Open(filename)
	defer f.Close()
	if err != nil {
		log.Panic(err)
	}

	_, err = f.Read(buffer)
	if err != nil {
		log.Panic(err)
	}

	return fmt.Sprintf("%d", crc64.Checksum(buffer, crc64.MakeTable(crc64.ISO)))
}
Beispiel #12
0
func newUserPermission(fields []proto.Field, values []sqltypes.Value) *UserPermission {
	up := &UserPermission{Privileges: make(map[string]string)}
	for i, field := range fields {
		switch field.Name {
		case "Host":
			up.Host = values[i].String()
		case "User":
			up.User = values[i].String()
		case "Password":
			up.PasswordChecksum = crc64.Checksum(([]byte)(values[i].String()), hashTable)
		default:
			up.Privileges[field.Name] = values[i].String()
		}
	}
	return up
}
Beispiel #13
0
// Lookup looks up a path
func (r *Root) Lookup(ctx context.Context, name string) (fs.Node, error) {
	logrus.WithField("name", name).Debug("handling Root.Lookup call")

	// TODO: handle context cancellation
	secret, err := r.logic.Read(path.Join(r.root, name))
	if secret == nil && err == nil {
		return nil, fuse.ENOENT
	} else if err != nil {
		logrus.WithError(err).WithFields(logrus.Fields{"root": r.root, "name": name}).Error("error reading key")
		return nil, fuse.EIO
	}

	return Secret{
		secret,
		crc64.Checksum([]byte(name), table),
	}, nil
}
Beispiel #14
0
func (o *JDict) Hash() int64 {
	panic("TODO")
	//#if 0
	var z int64
	//#if mudict
	o.mu.Lock()
	//#endif
	for k, v := range o.ppp {
		z += int64(crc64.Checksum([]byte(k), CrcPolynomial))
		z += v.Hash() // TODO better
	}
	//#if mudict
	o.mu.Unlock()
	//#endif
	return z
	//#endif
}
Beispiel #15
0
func TestNextPowerOf2(t *testing.T) {
	test := []struct {
		n    uint64
		next uint64
	}{
		{0, 0},
		{1, 1},
		{2, 2},
		{3, 2},
		{5, 3},
		{16, 5},
		{1 << 33, 34},
	}
	t.Log(crc64.Checksum([]byte("Paul Taysom"), hashTable))
	for _, v := range test {
		if nextPowerOf2(v.n) != v.next {
			t.Errorf("nextPowerOf2(%d) != %d\n", v.n, v.next)
		}
	}
}
Beispiel #16
0
func verifyEntry(entry string) error {
	fields := strings.SplitN(entry, " ", 2)
	if len(fields) < 2 {
		return fmt.Errorf("invalid entry format")
	}

	strHash := fields[0]
	strData := fields[1]

	intHash, err := strconv.ParseUint(strHash, 0, 64)
	if err != nil {
		return err
	}

	gotHash := crc.Checksum([]byte(strData), crc.MakeTable(crc.ECMA))
	if gotHash != intHash {
		return fmt.Errorf("hash mismatch: expected %v got %v", intHash, gotHash)
	}

	return nil
}
Beispiel #17
0
func hash(b []byte) uint64 {
	return crc64.Checksum(b, hashTable)
}
Beispiel #18
0
func serve(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, fmt.Sprintf("This resource does not accept %s requests.", r.Method), http.StatusMethodNotAllowed)
		return
	}

	domain := r.URL.Path[1:]

	h := w.Header()
	h.Set("Content-Type", "application/json; charset=utf-8")
	h.Set("Cache-Control", "public, max-age=900")
	h.Set("Access-Control-Allow-Origin", "*")

	srvFound := true
	_, srv, err := net.LookupSRV("xmpp-client", "tcp", domain)
	if err != nil {
		if !strings.HasSuffix(err.Error(), "DNS name does not exist.") {
			log.Printf("Error resolving SRV records for %q: %v", domain, err)
			httpError(w, internalServerError, http.StatusInternalServerError)
			return
		}

		srvFound = false
	}

	txtFound := true
	txt, err := net.LookupTXT("_xmppconnect." + domain)
	if err != nil {
		if !strings.HasSuffix(err.Error(), "DNS name does not exist.") {
			log.Printf("Error resolving TXT records for %q: %v", domain, err)
			httpError(w, internalServerError, http.StatusInternalServerError)
			return
		}

		txtFound = false
	}

	if !txtFound && !srvFound {
		httpError(w, notFoundError, http.StatusNotFound)
		return
	}

	res := &response{
		Version: "1.0",

		Data: &struct {
			Servers      serverList      `json:"servers"`
			Alternatives alternativeList `json:"alternatives"`
		}{
			Servers:      make([]*server, len(srv)),
			Alternatives: make([]*alternative, len(txt)),
		},
	}

	for i, service := range srv {
		res.Data.Servers[i] = &server{
			Target:   service.Target,
			Port:     service.Port,
			Priority: service.Priority,
			Weight:   service.Weight,
		}
	}

	for i, rec := range txt {
		split := strings.SplitN(rec, "=", 2)

		name := split[0]
		if !strings.HasPrefix(strings.ToLower(name), "_xmpp-client-") {
			continue
		}

		name = name[13:]

		res.Data.Alternatives[i] = &alternative{
			Name:  name,
			Value: split[1],
		}
	}

	if len(res.Data.Servers) == 0 && len(res.Data.Alternatives) == 0 {
		httpError(w, notFoundError, http.StatusNotFound)
		return
	}

	sort.Sort(res.Data.Servers)
	sort.Sort(res.Data.Alternatives)

	encoded, err := json.Marshal(res)
	if err != nil {
		log.Fatalf("Error marshalling JSON for %q: %v", domain, err)
	}

	hash := crc64.Checksum(encoded, crcTable)

	h.Set("ETag", "\""+strconv.FormatUint(hash, 16)+"\"")

	content := bytes.NewReader(encoded)
	http.ServeContent(w, r, domain, time.Time{}, content)
}
Beispiel #19
0
func SendEmail(c appengine.Context, key *datastore.Key) error {
	// Lookup the publication
	var pub Publication
	c.Infof("Using key %s", key.StringID())
	if err := datastore.Get(c, key, &pub); err != nil {
		return err
	}

	// Update the mod times
	pub.LastChecked = time.Now()

	// Go and get the content
	client := urlfetch.Client(c)
	resp, err := client.Get(pub.Url)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return &UrlFetchError{key.IntID(), pub.Url, resp.Status}
	}

	buf := make([]byte, resp.ContentLength)
	_, err = resp.Body.Read(buf)
	if err != nil {
		return err
	}

	checksum := strconv.FormatUint(crc64.Checksum(buf, crc64.MakeTable(crc64.ISO)), 36)
	if checksum == pub.Checksum {
		c.Infof("Checksum has not changed.  Not sending mail.")
		return nil
	}

	c.Infof("Checksum for [%s] has changed.  Sending mail.", pub.Name)
	pub.Length = resp.ContentLength
	pub.LastChanged = time.Now()
	pub.Checksum = checksum

	doc := &mail.Attachment{
		Name: "Update from Enbook.me",
		Data: buf,
	}
	message := &mail.Message{
		Sender:      "Enbook.me <*****@*****.**>",
		To:          []string{pub.Email},
		Subject:     "",
		Body:        "Update from Enbook.me",
		Attachments: []mail.Attachment{*doc},
	}
	if err := mail.Send(c, message); err != nil {
		c.Errorf("Failed to send mail [%s].", pub.Name)
		return err
	}
	pub.LastUpdateSent = time.Now()

	// Save the changes
	if _, err = datastore.Put(c, key, &pub); err != nil {
		c.Errorf("Failed to save datastore entity [%s].", pub.Name)
		return err
	}
	return nil
}
Beispiel #20
0
func HashCode(data string) uint64 {
	crcTable := crc64.MakeTable(crc64.ECMA)
	return crc64.Checksum([]byte(data), crcTable)
}
	xxhash "github.com/OneOfOne/xxhash/native"
	"github.com/dchest/siphash"
	farm "github.com/dgryski/go-farm"
	metro "github.com/dgryski/go-metro"
	spooky "github.com/dgryski/go-spooky"
	"github.com/opennota/fasthash"
	"github.com/zhenjl/cityhash"
	"hash/crc64"
	"hash/fnv"
)

var buf = make([]byte, 8192)

var ecmaTable = crc64.MakeTable(crc64.ECMA)
var hcrcecma = func(p []byte) uint64 { return crc64.Checksum(p, ecmaTable) }

func BenchmarkCRCECMA8(b *testing.B)   { benchmarkHash8(b, hcrcecma) }
func BenchmarkCRCECMA16(b *testing.B)  { benchmarkHash16(b, hcrcecma) }
func BenchmarkCRCECMA40(b *testing.B)  { benchmarkHash40(b, hcrcecma) }
func BenchmarkCRCECMA64(b *testing.B)  { benchmarkHash64(b, hcrcecma) }
func BenchmarkCRCECMA128(b *testing.B) { benchmarkHash128(b, hcrcecma) }
func BenchmarkCRCECMA1K(b *testing.B)  { benchmarkHash1K(b, hcrcecma) }
func BenchmarkCRCECMA8K(b *testing.B)  { benchmarkHash8K(b, hcrcecma) }

var isoTable = crc64.MakeTable(crc64.ISO)
var hcrciso = func(p []byte) uint64 { return crc64.Checksum(p, isoTable) }

func BenchmarkCRCISO8(b *testing.B)   { benchmarkHash8(b, hcrciso) }
func BenchmarkCRCISO16(b *testing.B)  { benchmarkHash16(b, hcrciso) }
func BenchmarkCRCISO40(b *testing.B)  { benchmarkHash40(b, hcrciso) }
Beispiel #22
0
// syntax sugar
func BufCRC64(bufr []byte) uint64 {
	return crc64.Checksum(bufr[:], G_crcTable)
}
Beispiel #23
0
func (s *RedisStore) bucketPartition(b []byte) string {
	check := crc64.Checksum(b, partitionTable)
	return fmt.Sprintf("%s.%d", partitionPrefix, check%s.MaxPartitions())
}
Beispiel #24
0
func (gitfs *GitFs) getGitAttrByPath(repoPath string, path string) (*fuse.Attr, fuse.Status) {
	repo, _, _, tree, err := gitfs.getMasterTreeFromRepo(repoPath)
	if err != nil {
		return nil, fuse.EPERM
	}

	repoInfo, err := os.Stat(repoPath)
	if err != nil {
		gitfs.logger.Debugf("Failed to Stat %s due to %s", repoPath, err)
		return nil, fuse.ToStatus(err)
	}

	if path == "" {
		attr := fuse.ToAttr(repoInfo)
		attr.Mode &= ^uint32(0222)
		attr.Nlink = 2 + gitfs.treeEntryCount(tree, repoPath)
		return attr, fuse.OK
	}

	entry, err := tree.EntryByPath(path)
	if err != nil {
		gitfs.logger.Debugf("Cannot find path %s from tree %s of Git Repository %s due to %s", path, tree.Id().String(), repoPath, err)
		return nil, fuse.ENOENT
	}
	gitfs.logger.Debugf("Found path %s from tree %s of Git Repository %s", path, tree.Id().String(), repoPath)

	var attr fuse.Attr
	attr.Mode = toFileMode(entry.Filemode)
	if stat, ok := repoInfo.Sys().(*syscall.Stat_t); ok {
		attr.Uid = stat.Uid
		attr.Gid = stat.Gid
		attr.Blksize = uint32(stat.Blksize)
		attr.Rdev = uint32(stat.Rdev)
	}
	attr.Ino = crc64.Checksum(entry.Id[:], crc64.MakeTable(crc64.ECMA))
	now := time.Now()
	attr.SetTimes(&now, &now, &now)

	switch entry.Type {
	case libgit2.ObjectTree:
		tree, err := repo.LookupTree(entry.Id)
		if err != nil {
			gitfs.logger.Errorf("Failed to find tree %s from Git Repository %s", entry.Id, repoPath)
			return nil, fuse.EPERM
		}
		defer tree.Free()
		gitfs.logger.Debugf("Found tree %s of Git Repository %s", tree.Id().String(), repoPath)
		attr.Size = 4096
		attr.Nlink = 2 + gitfs.treeEntryCount(tree, repoPath)
	case libgit2.ObjectBlob:
		blob, err := repo.LookupBlob(entry.Id)
		if err != nil {
			gitfs.logger.Errorf("Failed to find blob %s from Git Repository %s", entry.Id, repoPath)
			return nil, fuse.EPERM
		}
		defer blob.Free()
		gitfs.logger.Debugf("Found blob %s of Git Repository %s", blob.Id().String(), repoPath)
		attr.Nlink = 1
		attr.Size = uint64(blob.Size())
	default:
		gitfs.logger.Debugf("GetAttr: Unsupported object type %s of %s from Git Repository %s", entry.Type.String(), entry.Id, repoPath)
		return nil, fuse.ENOENT
	}
	attr.Blocks = (attr.Size + 511) / 512
	return &attr, fuse.OK
}
Beispiel #25
0
func TestCrc(t *testing.T) {
	t.Log(crc64.Checksum([]byte("Paul Taysom"), hashTable))
}
Beispiel #26
0
func ComputeRecordToken(record map[string]string) uint64 {
	crcTable := crc64.MakeTable(crc64.ECMA)
	return crc64.Checksum([]byte(fmt.Sprintf("%v", record)), crcTable)
}
Beispiel #27
0
func ComputeEntryToken(key string, value string) uint64 {
	crcTable := crc64.MakeTable(crc64.ECMA)
	return crc64.Checksum([]byte(fmt.Sprintf("%s:%s", key, value)), crcTable)
}
Beispiel #28
0
func (b *Bucket) Partition(id []byte, partitions uint64) uint64 {
	check := crc64.Checksum(id, PartitionTable)
	return check % partitions
}
Beispiel #29
0
func (label *Label) hash() {
	label.h = crc64.Checksum(label.data[:], labelCRC64Table)
}
Beispiel #30
0
// inode calculates the inode number of a given key as the CRC64 checksum.
func inode(key string) uint64 {
	return crc64.Checksum([]byte(key), crcTable)
}