func (id *Id) Partition(max uint64) uint64 {
b, err := id.Encode()
if err != nil {
panic(err)
}
return crc64.Checksum(b, partitionTable) % max
}
func BenchmarkCRC64ISOShort(b *testing.B) {
var bv uint64
k := []byte("Test-key-100")
for i := 0; i < b.N; i++ {
bv += crc64.Checksum(k, crc64ISO)
}
}
func (m M) Hash() int64 {
if m.X != nil {
return m.X.Hash()
} else if len(m.S) == 0 {
return m.N
}
return int64(crc64.Checksum([]byte(m.S), CrcPolynomial))
}
// HashVbuuid return crc64 value of list of 64-bit vbuuids.
func HashVbuuid(vbuuids []uint64) uint64 {
var bytes []byte
vbuuids_sl := (*reflect.SliceHeader)(unsafe.Pointer(&vbuuids))
bytes_sl := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
bytes_sl.Data = vbuuids_sl.Data
bytes_sl.Len = vbuuids_sl.Len * 8
bytes_sl.Cap = vbuuids_sl.Cap * 8
return crc64.Checksum(bytes, crc64.MakeTable(crc64.ECMA))
}
func FileCRC64(filenameOrURL string) (uint64, error) {
data, err := FileGetBytes(filenameOrURL)
if err != nil {
return 0, err
}
if crc64Table == nil {
crc64Table = crc64.MakeTable(crc64.ECMA)
}
return crc64.Checksum(data, crc64Table), nil
}
func (h *crcHash) Sum64() uint64 {
sort.Sort(h.factors)
data := make([]byte, len(h.factors)*8+1)
data[0] = h.prefix
for i, factor := range h.factors {
binary.LittleEndian.PutUint64(data[i*8+1:], factor)
}
return crc64.Checksum(data, crcTable)
}
// compute a hashcode for ExecutorInfo that may be used as a reasonable litmus test
// with respect to compatibility across HA schedulers. the intent is that an HA scheduler
// should fail-fast if it doesn't pass this test, rather than generating (potentially many)
// errors at run-time because a Mesos master decides that the ExecutorInfo generated by a
// secondary scheduler doesn't match that of the primary scheduler.
//
// Note: We intentionally leave out the Resources in this hash because they are
// set during procurement and should not lead to a different ExecutorId.
// This also means that the Resources do not contribute to offer
// compatibility checking. But as we persist and restore the Resources
// through node anotation we make sure that the right resources are chosen
// during task launch.
//
// see https://github.com/apache/mesos/blob/0.22.0/src/common/type_utils.cpp#L110
func hash(info *mesos.ExecutorInfo) uint64 {
// !!! we specifically do NOT include:
// - Framework ID because it's a value that's initialized too late for us to use
// - Executor ID because it's a value that includes a copy of this hash
buf := &bytes.Buffer{}
buf.WriteString(info.GetName())
buf.WriteString(info.GetSource())
buf.Write(info.Data)
if info.Command != nil {
buf.WriteString(info.Command.GetValue())
buf.WriteString(info.Command.GetUser())
buf.WriteString(strconv.FormatBool(info.Command.GetShell()))
if sz := len(info.Command.Arguments); sz > 0 {
x := make([]string, sz)
copy(x, info.Command.Arguments)
sort.Strings(x)
for _, item := range x {
buf.WriteString(item)
}
}
if vars := info.Command.Environment.GetVariables(); len(vars) > 0 {
names := []string{}
e := make(map[string]string)
for _, v := range vars {
if name := v.GetName(); name != "" {
names = append(names, name)
e[name] = v.GetValue()
}
}
sort.Strings(names)
for _, n := range names {
buf.WriteString(n)
buf.WriteString("=")
buf.WriteString(e[n])
}
}
if uris := info.Command.GetUris(); len(uris) > 0 {
su := []string{}
for _, uri := range uris {
su = append(su, fmt.Sprintf("%s%t%t", uri.GetValue(), uri.GetExecutable(), uri.GetExtract()))
}
sort.Strings(su)
for _, uri := range su {
buf.WriteString(uri)
}
}
//TODO(jdef) add support for Container
}
table := crc64.MakeTable(crc64.ECMA)
return crc64.Checksum(buf.Bytes(), table)
}
// NewRand returns a seeded random number generator, using a seed derived
// from the provided string.
//
// If the seed string is empty, the current time is used as a seed.
func NewRand(seed string) *rand.Rand {
var seedInt int64
if seed != "" {
crcTable := crc64.MakeTable(crc64.ISO)
seedInt = int64(crc64.Checksum([]byte(seed), crcTable))
} else {
seedInt = time.Now().Unix()
}
randSource := rand.NewSource(seedInt)
return rand.New(randSource)
}
// WorBig generates string for multi-precision integer n, count of words
// calculated as:
// (bitLength(n) / 16) rounded towards infinity
func WorBig(n *big.Int) string {
switch {
case n.BitLen() <= 16:
return Wor(n.Int64(), 1)
case n.BitLen() <= 32:
return Wor(n.Int64(), 2)
case n.BitLen() <= 64:
return Wor(n.Int64(), 4)
default:
sum := crc64.Checksum(n.Bytes(), crc64.MakeTable(crc64.ECMA))
return Wor(int64(sum), (n.BitLen()+16)/16)
}
}
func (v *VagrantShareRemoteWatcher) downloadJson(table *crc64.Table) (uint64, *VagrantBoxDescripter, error) {
body, err := v.jsonGetter(v.url)
if err != nil {
return 0, nil, err
}
checksum64 := crc64.Checksum(body, table)
var descripter VagrantBoxDescripter
json.Unmarshal(body, &descripter)
return checksum64, &descripter, nil
}
func calculateCRC64Hash(filename string) string {
var buffer []byte = make([]byte, 1024**NumberOfBytesToRead)
f, err := os.Open(filename)
defer f.Close()
if err != nil {
log.Panic(err)
}
_, err = f.Read(buffer)
if err != nil {
log.Panic(err)
}
return fmt.Sprintf("%d", crc64.Checksum(buffer, crc64.MakeTable(crc64.ISO)))
}
func newUserPermission(fields []proto.Field, values []sqltypes.Value) *UserPermission {
up := &UserPermission{Privileges: make(map[string]string)}
for i, field := range fields {
switch field.Name {
case "Host":
up.Host = values[i].String()
case "User":
up.User = values[i].String()
case "Password":
up.PasswordChecksum = crc64.Checksum(([]byte)(values[i].String()), hashTable)
default:
up.Privileges[field.Name] = values[i].String()
}
}
return up
}
// Lookup looks up a path
func (r *Root) Lookup(ctx context.Context, name string) (fs.Node, error) {
logrus.WithField("name", name).Debug("handling Root.Lookup call")
// TODO: handle context cancellation
secret, err := r.logic.Read(path.Join(r.root, name))
if secret == nil && err == nil {
return nil, fuse.ENOENT
} else if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{"root": r.root, "name": name}).Error("error reading key")
return nil, fuse.EIO
}
return Secret{
secret,
crc64.Checksum([]byte(name), table),
}, nil
}
func (o *JDict) Hash() int64 {
panic("TODO")
//#if 0
var z int64
//#if mudict
o.mu.Lock()
//#endif
for k, v := range o.ppp {
z += int64(crc64.Checksum([]byte(k), CrcPolynomial))
z += v.Hash() // TODO better
}
//#if mudict
o.mu.Unlock()
//#endif
return z
//#endif
}
func TestNextPowerOf2(t *testing.T) {
test := []struct {
n uint64
next uint64
}{
{0, 0},
{1, 1},
{2, 2},
{3, 2},
{5, 3},
{16, 5},
{1 << 33, 34},
}
t.Log(crc64.Checksum([]byte("Paul Taysom"), hashTable))
for _, v := range test {
if nextPowerOf2(v.n) != v.next {
t.Errorf("nextPowerOf2(%d) != %d\n", v.n, v.next)
}
}
}
func verifyEntry(entry string) error {
fields := strings.SplitN(entry, " ", 2)
if len(fields) < 2 {
return fmt.Errorf("invalid entry format")
}
strHash := fields[0]
strData := fields[1]
intHash, err := strconv.ParseUint(strHash, 0, 64)
if err != nil {
return err
}
gotHash := crc.Checksum([]byte(strData), crc.MakeTable(crc.ECMA))
if gotHash != intHash {
return fmt.Errorf("hash mismatch: expected %v got %v", intHash, gotHash)
}
return nil
}
func hash(b []byte) uint64 {
return crc64.Checksum(b, hashTable)
}
func serve(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, fmt.Sprintf("This resource does not accept %s requests.", r.Method), http.StatusMethodNotAllowed)
return
}
domain := r.URL.Path[1:]
h := w.Header()
h.Set("Content-Type", "application/json; charset=utf-8")
h.Set("Cache-Control", "public, max-age=900")
h.Set("Access-Control-Allow-Origin", "*")
srvFound := true
_, srv, err := net.LookupSRV("xmpp-client", "tcp", domain)
if err != nil {
if !strings.HasSuffix(err.Error(), "DNS name does not exist.") {
log.Printf("Error resolving SRV records for %q: %v", domain, err)
httpError(w, internalServerError, http.StatusInternalServerError)
return
}
srvFound = false
}
txtFound := true
txt, err := net.LookupTXT("_xmppconnect." + domain)
if err != nil {
if !strings.HasSuffix(err.Error(), "DNS name does not exist.") {
log.Printf("Error resolving TXT records for %q: %v", domain, err)
httpError(w, internalServerError, http.StatusInternalServerError)
return
}
txtFound = false
}
if !txtFound && !srvFound {
httpError(w, notFoundError, http.StatusNotFound)
return
}
res := &response{
Version: "1.0",
Data: &struct {
Servers serverList `json:"servers"`
Alternatives alternativeList `json:"alternatives"`
}{
Servers: make([]*server, len(srv)),
Alternatives: make([]*alternative, len(txt)),
},
}
for i, service := range srv {
res.Data.Servers[i] = &server{
Target: service.Target,
Port: service.Port,
Priority: service.Priority,
Weight: service.Weight,
}
}
for i, rec := range txt {
split := strings.SplitN(rec, "=", 2)
name := split[0]
if !strings.HasPrefix(strings.ToLower(name), "_xmpp-client-") {
continue
}
name = name[13:]
res.Data.Alternatives[i] = &alternative{
Name: name,
Value: split[1],
}
}
if len(res.Data.Servers) == 0 && len(res.Data.Alternatives) == 0 {
httpError(w, notFoundError, http.StatusNotFound)
return
}
sort.Sort(res.Data.Servers)
sort.Sort(res.Data.Alternatives)
encoded, err := json.Marshal(res)
if err != nil {
log.Fatalf("Error marshalling JSON for %q: %v", domain, err)
}
hash := crc64.Checksum(encoded, crcTable)
h.Set("ETag", "\""+strconv.FormatUint(hash, 16)+"\"")
content := bytes.NewReader(encoded)
http.ServeContent(w, r, domain, time.Time{}, content)
}
func SendEmail(c appengine.Context, key *datastore.Key) error {
// Lookup the publication
var pub Publication
c.Infof("Using key %s", key.StringID())
if err := datastore.Get(c, key, &pub); err != nil {
return err
}
// Update the mod times
pub.LastChecked = time.Now()
// Go and get the content
client := urlfetch.Client(c)
resp, err := client.Get(pub.Url)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return &UrlFetchError{key.IntID(), pub.Url, resp.Status}
}
buf := make([]byte, resp.ContentLength)
_, err = resp.Body.Read(buf)
if err != nil {
return err
}
checksum := strconv.FormatUint(crc64.Checksum(buf, crc64.MakeTable(crc64.ISO)), 36)
if checksum == pub.Checksum {
c.Infof("Checksum has not changed. Not sending mail.")
return nil
}
c.Infof("Checksum for [%s] has changed. Sending mail.", pub.Name)
pub.Length = resp.ContentLength
pub.LastChanged = time.Now()
pub.Checksum = checksum
doc := &mail.Attachment{
Name: "Update from Enbook.me",
Data: buf,
}
message := &mail.Message{
Sender: "Enbook.me <*****@*****.**>",
To: []string{pub.Email},
Subject: "",
Body: "Update from Enbook.me",
Attachments: []mail.Attachment{*doc},
}
if err := mail.Send(c, message); err != nil {
c.Errorf("Failed to send mail [%s].", pub.Name)
return err
}
pub.LastUpdateSent = time.Now()
// Save the changes
if _, err = datastore.Put(c, key, &pub); err != nil {
c.Errorf("Failed to save datastore entity [%s].", pub.Name)
return err
}
return nil
}
func HashCode(data string) uint64 {
crcTable := crc64.MakeTable(crc64.ECMA)
return crc64.Checksum([]byte(data), crcTable)
}
xxhash "github.com/OneOfOne/xxhash/native"
"github.com/dchest/siphash"
farm "github.com/dgryski/go-farm"
metro "github.com/dgryski/go-metro"
spooky "github.com/dgryski/go-spooky"
"github.com/opennota/fasthash"
"github.com/zhenjl/cityhash"
"hash/crc64"
"hash/fnv"
)
var buf = make([]byte, 8192)
var ecmaTable = crc64.MakeTable(crc64.ECMA)
var hcrcecma = func(p []byte) uint64 { return crc64.Checksum(p, ecmaTable) }
func BenchmarkCRCECMA8(b *testing.B) { benchmarkHash8(b, hcrcecma) }
func BenchmarkCRCECMA16(b *testing.B) { benchmarkHash16(b, hcrcecma) }
func BenchmarkCRCECMA40(b *testing.B) { benchmarkHash40(b, hcrcecma) }
func BenchmarkCRCECMA64(b *testing.B) { benchmarkHash64(b, hcrcecma) }
func BenchmarkCRCECMA128(b *testing.B) { benchmarkHash128(b, hcrcecma) }
func BenchmarkCRCECMA1K(b *testing.B) { benchmarkHash1K(b, hcrcecma) }
func BenchmarkCRCECMA8K(b *testing.B) { benchmarkHash8K(b, hcrcecma) }
var isoTable = crc64.MakeTable(crc64.ISO)
var hcrciso = func(p []byte) uint64 { return crc64.Checksum(p, isoTable) }
func BenchmarkCRCISO8(b *testing.B) { benchmarkHash8(b, hcrciso) }
func BenchmarkCRCISO16(b *testing.B) { benchmarkHash16(b, hcrciso) }
func BenchmarkCRCISO40(b *testing.B) { benchmarkHash40(b, hcrciso) }
// syntax sugar
func BufCRC64(bufr []byte) uint64 {
return crc64.Checksum(bufr[:], G_crcTable)
}
func (s *RedisStore) bucketPartition(b []byte) string {
check := crc64.Checksum(b, partitionTable)
return fmt.Sprintf("%s.%d", partitionPrefix, check%s.MaxPartitions())
}
func (gitfs *GitFs) getGitAttrByPath(repoPath string, path string) (*fuse.Attr, fuse.Status) {
repo, _, _, tree, err := gitfs.getMasterTreeFromRepo(repoPath)
if err != nil {
return nil, fuse.EPERM
}
repoInfo, err := os.Stat(repoPath)
if err != nil {
gitfs.logger.Debugf("Failed to Stat %s due to %s", repoPath, err)
return nil, fuse.ToStatus(err)
}
if path == "" {
attr := fuse.ToAttr(repoInfo)
attr.Mode &= ^uint32(0222)
attr.Nlink = 2 + gitfs.treeEntryCount(tree, repoPath)
return attr, fuse.OK
}
entry, err := tree.EntryByPath(path)
if err != nil {
gitfs.logger.Debugf("Cannot find path %s from tree %s of Git Repository %s due to %s", path, tree.Id().String(), repoPath, err)
return nil, fuse.ENOENT
}
gitfs.logger.Debugf("Found path %s from tree %s of Git Repository %s", path, tree.Id().String(), repoPath)
var attr fuse.Attr
attr.Mode = toFileMode(entry.Filemode)
if stat, ok := repoInfo.Sys().(*syscall.Stat_t); ok {
attr.Uid = stat.Uid
attr.Gid = stat.Gid
attr.Blksize = uint32(stat.Blksize)
attr.Rdev = uint32(stat.Rdev)
}
attr.Ino = crc64.Checksum(entry.Id[:], crc64.MakeTable(crc64.ECMA))
now := time.Now()
attr.SetTimes(&now, &now, &now)
switch entry.Type {
case libgit2.ObjectTree:
tree, err := repo.LookupTree(entry.Id)
if err != nil {
gitfs.logger.Errorf("Failed to find tree %s from Git Repository %s", entry.Id, repoPath)
return nil, fuse.EPERM
}
defer tree.Free()
gitfs.logger.Debugf("Found tree %s of Git Repository %s", tree.Id().String(), repoPath)
attr.Size = 4096
attr.Nlink = 2 + gitfs.treeEntryCount(tree, repoPath)
case libgit2.ObjectBlob:
blob, err := repo.LookupBlob(entry.Id)
if err != nil {
gitfs.logger.Errorf("Failed to find blob %s from Git Repository %s", entry.Id, repoPath)
return nil, fuse.EPERM
}
defer blob.Free()
gitfs.logger.Debugf("Found blob %s of Git Repository %s", blob.Id().String(), repoPath)
attr.Nlink = 1
attr.Size = uint64(blob.Size())
default:
gitfs.logger.Debugf("GetAttr: Unsupported object type %s of %s from Git Repository %s", entry.Type.String(), entry.Id, repoPath)
return nil, fuse.ENOENT
}
attr.Blocks = (attr.Size + 511) / 512
return &attr, fuse.OK
}
func TestCrc(t *testing.T) {
t.Log(crc64.Checksum([]byte("Paul Taysom"), hashTable))
}
func ComputeRecordToken(record map[string]string) uint64 {
crcTable := crc64.MakeTable(crc64.ECMA)
return crc64.Checksum([]byte(fmt.Sprintf("%v", record)), crcTable)
}
func ComputeEntryToken(key string, value string) uint64 {
crcTable := crc64.MakeTable(crc64.ECMA)
return crc64.Checksum([]byte(fmt.Sprintf("%s:%s", key, value)), crcTable)
}
func (b *Bucket) Partition(id []byte, partitions uint64) uint64 {
check := crc64.Checksum(id, PartitionTable)
return check % partitions
}
func (label *Label) hash() {
label.h = crc64.Checksum(label.data[:], labelCRC64Table)
}
// inode calculates the inode number of a given key as the CRC64 checksum.
func inode(key string) uint64 {
return crc64.Checksum([]byte(key), crcTable)
}