func CheckECC(data []byte) uint32 { if len(data) > 200 { tmpdata := append(data[:100], data[len(data)-100:]...) return adler32.Checksum(tmpdata) } else { return adler32.Checksum(data) } }
// FIXME: find better way of writing this func Encode(msg *RpcMessage) ([]byte, error) { payload, err := proto.Marshal(msg) if err != nil { return nil, err } wire := make([]byte, 12+len(payload)) // size binary.BigEndian.PutUint32(wire, uint32(8+len(payload))) // marker if copy(wire[4:], "RPC0") != 4 { panic("What the hell") } // payload if copy(wire[8:], payload) != len(payload) { panic("What the hell") } // checksum checksum := adler32.Checksum(wire[4 : 8+len(payload)]) binary.BigEndian.PutUint32(wire[8+len(payload):], checksum) return wire, nil }
func BenchmarkAdler32(b *testing.B) { var bv uint32 for i := 0; i < b.N; i++ { bv = adler32.Checksum(in) } benchVal32 = bv }
func (this *StandardPeerSelector) PickPeer(key string) (peerAddr string) { // adler32 is almost same as crc32, but much 3 times faster checksum := adler32.Checksum([]byte(key)) index := int(checksum) % len(this.peerAddrs) return this.peerAddrs[index] }
// performReducing runs the reducing goroutines. func performReducing(mr MapReducer, mapEmitChan, reduceEmitChan KeyValueChan) { // Start a closer for the reduce emit chan. size := runtime.NumCPU() signals := newCloserChan(reduceEmitChan, size) // Start reduce goroutines. reduceChans := make([]KeyValueChan, size) for i := 0; i < size; i++ { reduceChans[i] = make(KeyValueChan) go func(in KeyValueChan) { mr.Reduce(in, reduceEmitChan) signals <- struct{}{} }(reduceChans[i]) } // Read map emitted data. for kv := range mapEmitChan { hash := adler32.Checksum([]byte(kv.Key())) idx := hash % uint32(size) reduceChans[idx] <- kv } // Close reduce channels. for _, reduceChan := range reduceChans { reduceChan.Close() } }
// NewReaderDict is like NewReader but uses a preset dictionary. // NewReaderDict ignores the dictionary if the compressed data does not refer to it. func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, os.Error) { z := new(reader) if fr, ok := r.(flate.Reader); ok { z.r = fr } else { z.r = bufio.NewReader(r) } _, err := io.ReadFull(z.r, z.scratch[0:2]) if err != nil { return nil, err } h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { return nil, HeaderError } if z.scratch[1]&0x20 != 0 { _, err = io.ReadFull(z.r, z.scratch[0:4]) if err != nil { return nil, err } checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) if checksum != adler32.Checksum(dict) { return nil, DictionaryError } z.decompressor = flate.NewReaderDict(z.r, dict) } else { z.decompressor = flate.NewReader(z.r) } z.digest = adler32.New() return z, nil }
func Decode(r io.Reader) (msg *RpcMessage, err error) { header := make([]byte, 4) _, err = io.ReadFull(r, header) if err != nil { return } length := binary.BigEndian.Uint32(header) payload := make([]byte, length) _, err = io.ReadFull(r, payload) if err != nil { return } if string(payload[:4]) != "RPC0" { err = fmt.Errorf("Wrong marker") return } checksum := adler32.Checksum(payload[:length-4]) if checksum != binary.BigEndian.Uint32(payload[length-4:]) { err = fmt.Errorf("Wrong checksum") return } msg = new(RpcMessage) err = proto.Unmarshal(payload[4:length-4], msg) return }
// Perform the reducing. func performReducing(mapEmitChan KeyValueChan, reduceFunc ReduceFunc, reduceSize int, reduceEmitChan KeyValueChan) { // Start a closer for the reduce emit chan. sigChan := closeSignalChannel(reduceEmitChan, reduceSize) // Start reduce funcs. reduceChans := make(KeyValueChans, reduceSize) for i := 0; i < reduceSize; i++ { reduceChans[i] = make(KeyValueChan) go func(inChan KeyValueChan) { reduceFunc(inChan, reduceEmitChan) sigChan <- true }(reduceChans[i]) } // Read map emitted data. for kv := range mapEmitChan { hash := adler32.Checksum([]byte(kv.Key)) idx := hash % uint32(reduceSize) reduceChans[idx] <- kv } // Close reduce channels. for _, reduceChan := range reduceChans { close(reduceChan) } }
func (section_header *Section_Header) Verify(datar *bytes.Reader) bool { var buf []byte datar.Read(buf) fmt.Println(section_header.Checksum, len(buf)) return section_header.Checksum == adler32.Checksum(buf[:72]) }
func errorClass(err error) string { class := reflect.TypeOf(err).String() if class == "" { return "panic" } else if class == "*errors.errorString" { checksum := adler32.Checksum([]byte(err.Error())) return fmt.Sprintf("{%x}", checksum) } else { return strings.TrimPrefix(class, "*") } }
// Put sets a value by key in the hash map func (t *HashTable) Put(key, value string) { hash := adler32.Checksum([]byte(key)) % maxTableSize for { if t.Table[hash] != nil && t.Table[hash].Key != key { hash = (hash + 1) % maxTableSize } else { break } } t.Table[hash] = &node{key, value} }
func (c *Catalog) Include(content []byte) (ret bool) { crc := adler32.Checksum([]byte(content)) sort.Sort(c.Files) exists := sort.Search(len(c.Files), func(i int) bool { return c.Files[i] >= crc }) if exists < len(c.Files) && c.Files[exists] == crc { return true } return false }
func BenchmarkAdler32(b *testing.B) { in := make([]byte, 10000) for i := 0; i < len(in); i++ { in[i] = byte(rand.Intn(255)) } b.ResetTimer() for i := 0; i < b.N; i++ { adler32.Checksum(in) } b.SetBytes(int64(len(in))) }
// writeHeader writes the ZLIB header. func (z *Writer) writeHeader() (err error) { z.wroteHeader = true // ZLIB has a two-byte header (as documented in RFC 1950). // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. // The next four bits is the CM (compression method), which is 8 for deflate. z.scratch[0] = 0x78 // The next two bits is the FLEVEL (compression level). The four values are: // 0=fastest, 1=fast, 2=default, 3=best. // The next bit, FDICT, is set if a dictionary is given. // The final five FCHECK bits form a mod-31 checksum. switch z.level { case 0, 1: z.scratch[1] = 0 << 6 case 2, 3, 4, 5: z.scratch[1] = 1 << 6 case 6, -1: z.scratch[1] = 2 << 6 case 7, 8, 9: z.scratch[1] = 3 << 6 default: panic("unreachable") } if z.dict != nil { z.scratch[1] |= 1 << 5 } z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) if _, err = z.w.Write(z.scratch[0:2]); err != nil { return err } if z.dict != nil { // The next four bytes are the Adler-32 checksum of the dictionary. checksum := adler32.Checksum(z.dict) z.scratch[0] = uint8(checksum >> 24) z.scratch[1] = uint8(checksum >> 16) z.scratch[2] = uint8(checksum >> 8) z.scratch[3] = uint8(checksum >> 0) if _, err = z.w.Write(z.scratch[0:4]); err != nil { return err } } if z.compressor == nil { // Initialize deflater unless the Writer is being reused // after a Reset call. z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) if err != nil { return err } z.digest = adler32.New() } return nil }
// StartJob launches a job on the given queue. It is not executed immediately but // scheduled to run as a task which performs splitting of the input reader based // on the number of shards. func (m *mapper) startJobHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) values := r.URL.Query() name := values.Get("name") jobSpec, err := CreateJobInstance(name) if err != nil { return } shards, err := strconv.Atoi(values.Get("shards")) if shards == 0 || err != nil { shards = m.config.Shards } queue := values.Get("queue") if queue != "" { // override the queue for this request // (used by locker.Schedule later) c = locker.WithQueue(c, queue) } bucket := values.Get("bucket") query, err := jobSpec.Query(r) if err != nil { log.Errorf(c, "error creating query %v", err) w.WriteHeader(http.StatusBadRequest) return } requestHash := r.Header.Get("X-Appengine-Request-Id-Hash") if requestHash == "" { // this should only happen when testing, we just need a short hash requestID := appengine.RequestID(c) requestHash = strconv.FormatUint(uint64(adler32.Checksum([]byte(requestID))), 16) } id := fmt.Sprintf("%s/%s", name, requestHash) job := &job{ JobName: name, JobSpec: jobSpec, Bucket: bucket, Shards: shards, Iterating: true, } job.common.start(query) key := datastore.NewKey(c, m.config.DatastorePrefix+jobKind, id, 0, nil) m.locker.Schedule(c, key, job, m.config.Path+jobURL, nil) }
// Get returns a value by string, and true/false if the key exists func (t HashTable) Get(key string) (string, bool) { hash := adler32.Checksum([]byte(key)) % maxTableSize for { if t.Table[hash] != nil && t.Table[hash].Key != key { hash = (hash + 1) % maxTableSize } else { break } } if t.Table[hash] == nil { return "", false } return t.Table[hash].Value, true }
func (z *reader) Reset(r io.Reader, dict []byte) error { *z = reader{decompressor: z.decompressor} if fr, ok := r.(flate.Reader); ok { z.r = fr } else { z.r = bufio.NewReader(r) } // Read the header (RFC 1950 section 2.2.). _, z.err = io.ReadFull(z.r, z.scratch[0:2]) if z.err != nil { if z.err == io.EOF { z.err = io.ErrUnexpectedEOF } return z.err } h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { z.err = ErrHeader return z.err } haveDict := z.scratch[1]&0x20 != 0 if haveDict { _, z.err = io.ReadFull(z.r, z.scratch[0:4]) if z.err != nil { if z.err == io.EOF { z.err = io.ErrUnexpectedEOF } return z.err } checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) if checksum != adler32.Checksum(dict) { z.err = ErrDictionary return z.err } } if z.decompressor == nil { if haveDict { z.decompressor = flate.NewReaderDict(z.r, dict) } else { z.decompressor = flate.NewReader(z.r) } } else { z.decompressor.(flate.Resetter).Reset(z.r, dict) } z.digest = adler32.New() return nil }
// NewWriterDict creates a new io.WriteCloser that satisfies writes by compressing data written to w. // It is the caller's responsibility to call Close on the WriteCloser when done. // level is the compression level, which can be DefaultCompression, NoCompression, // or any integer value between BestSpeed and BestCompression (inclusive). // dict is the preset dictionary to compress with, or nil to use no dictionary. func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, os.Error) { z := new(Writer) // ZLIB has a two-byte header (as documented in RFC 1950). // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. // The next four bits is the CM (compression method), which is 8 for deflate. z.scratch[0] = 0x78 // The next two bits is the FLEVEL (compression level). The four values are: // 0=fastest, 1=fast, 2=default, 3=best. // The next bit, FDICT, is set if a dictionary is given. // The final five FCHECK bits form a mod-31 checksum. switch level { case 0, 1: z.scratch[1] = 0 << 6 case 2, 3, 4, 5: z.scratch[1] = 1 << 6 case 6, -1: z.scratch[1] = 2 << 6 case 7, 8, 9: z.scratch[1] = 3 << 6 default: return nil, os.NewError("level out of range") } if dict != nil { z.scratch[1] |= 1 << 5 } z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) _, err := w.Write(z.scratch[0:2]) if err != nil { return nil, err } if dict != nil { // The next four bytes are the Adler-32 checksum of the dictionary. checksum := adler32.Checksum(dict) z.scratch[0] = uint8(checksum >> 24) z.scratch[1] = uint8(checksum >> 16) z.scratch[2] = uint8(checksum >> 8) z.scratch[3] = uint8(checksum >> 0) _, err = w.Write(z.scratch[0:4]) if err != nil { return nil, err } } z.w = w z.compressor = flate.NewWriterDict(w, level, dict) z.digest = adler32.New() return z, nil }
func main() { servers := make([]*store.StoreServer, NUM_SHARDS) i := 0 for i = 0; i < NUM_SHARDS; i++ { servers[i] = store.NewStoreServer() } //server := store.NewStoreServer() var e entry var firstTime uint64 scanner := bufio.NewScanner(os.Stdin) c := 0 n := 0 for scanner.Scan() { err := json.Unmarshal([]byte(scanner.Text()), &e) if err != nil { n += 1 continue } if firstTime == 0 { firstTime = e.Time } name := e.timeSeriesName() shard := adler32.Checksum([]byte(name)) % NUM_SHARDS servers[shard].Put(e.timeSeriesName(), uint32(e.Time/1000), e.Value) c += 1 } fmt.Printf("first_time: %d last_time: %d delta: %d\n", firstTime/1000, e.Time/1000, (e.Time-firstTime)/1000) total_bytes, total_series := uint64(0), 0 for i = 0; i < NUM_SHARDS; i++ { bytes_consumed := servers[i].BytesConsumed() num_time_series := servers[i].NumTimeSeries() fmt.Printf("%02d: bytes=%d #series=%d\n", i, bytes_consumed, num_time_series) total_bytes += uint64(bytes_consumed) total_series += num_time_series } fmt.Printf("total: bytes: %d series: %d count: %d\n", total_bytes, total_series, c) //router := mux.NewRouter().StrictSlash(true) //router.HandleFunc("/query", server.Query) //log.Fatal(http.ListenAndServe(":8080", router)) }
func TestOverflow(t *testing.T) { data := make([]byte, 65536) for k, _ := range data { data[k] = 255 } for i := 17343; i <= 17343; i++ { d := uint32(1) d = pushBack(d, data[:i]) if uint32(d) != adler32.Checksum(data[:i]) { t.Fatalf("pushBack seems to be wrong") } d = popFront(d, data[:i], i) if d != uint32(1) { t.Errorf("Overflow at length %d detected: d=%08x", i, d) } } }
func (e *partitionEmitter) Emit(reduceKey string, sortKey string, value string) { partition := uint32(0) if e.partitions > 1 { partition = adler32.Checksum([]byte(reduceKey)) % uint32(e.partitions) } if e.emitters[partition] == nil { e.FileNames[partition] = fmt.Sprintf("%s.%04d", e.fileNameTemplate, partition) fd, _ := os.Create(e.FileNames[partition]) e.fds[partition] = fd w := bufio.NewWriter(fd) e.emitters[partition] = newPrintEmitter(w) } e.emitters[partition].Emit(reduceKey, sortKey, value) }
func TestDecodeEncode(t *testing.T) { buf := bytes.NewBuffer([]byte(armorExample1)) result, err := Decode(buf) if err != nil { t.Error(err) } expectedType := "PGP SIGNATURE" if result.Type != expectedType { t.Errorf("result.Type: got:%s want:%s", result.Type, expectedType) } if len(result.Header) != 1 { t.Errorf("len(result.Header): got:%d want:1", len(result.Header)) } v, ok := result.Header["Version"] if !ok || v != "GnuPG v1.4.10 (GNU/Linux)" { t.Errorf("result.Header: got:%#v", result.Header) } contents, err := ioutil.ReadAll(result.Body) if err != nil { t.Error(err) } if adler32.Checksum(contents) != 0x789d7f00 { t.Errorf("contents: got: %x", contents) } buf = bytes.NewBuffer(nil) w, err := Encode(buf, result.Type, result.Header) if err != nil { t.Error(err) } _, err = w.Write(contents) if err != nil { t.Error(err) } w.Close() if !bytes.Equal(buf.Bytes(), []byte(armorExample1)) { t.Errorf("got: %s\nwant: %s", string(buf.Bytes()), armorExample1) } }
// Writes up to 4MB of random data into the rolling checksum func TestEquivalenceLong(t *testing.T) { testarr := make([]byte, 4*1024*1024) sum := 0 for sum != len(testarr) { num, _ := rand.Read(testarr[sum:]) sum += num } for window := 32; window <= 40960; window *= 2 { var rolling = New(uint32(window)) rolling.Write(testarr) start := len(testarr) - window expected := adler32.Checksum(testarr[start:]) actual := rolling.Sum32() if expected != actual { t.Fatalf("%d: expected %x, got %x", window, expected, actual) } } }
func (this *mysqlStore) KafkaTopic(appid string, topic string, ver string) (r string) { b := mpool.BytesBufferGet() b.Reset() b.WriteString(appid) b.WriteByte('.') b.WriteString(topic) b.WriteByte('.') b.WriteString(ver) if len(ver) > 2 { // ver starts with 'v1', from 'v10' on, will use obfuscation b.WriteByte('.') // can't use app secret as part of cookie: what if user changes his secret? // FIXME user can guess the cookie if they know the algorithm in advance cookie := adler32.Checksum([]byte(appid + topic)) b.WriteString(strconv.Itoa(int(cookie % 1000))) } r = b.String() mpool.BytesBufferPut(b) return }
//HTTP Request Handler func HandleRequest(rw http.ResponseWriter, req *http.Request) { //Get adjusted filepath which will ensure proper directory cleanpath := op.Path + filepath.Clean(req.URL.Path) //Get the checksum of the path n := adler32.Checksum([]byte(cleanpath))%40 + 1 //n = 9 strurl := "http://virtual" + fmt.Sprintf("%d", n) + ".cs.missouri.edu:8006" fmt.Println(strurl) u, err := url.Parse(strurl) if err != nil { log.Fatal(err) } //Create a reverse proxy reverse_proxy := httputil.NewSingleHostReverseProxy(u) reverse_proxy.ServeHTTP(rw, req) }
func checksum(r io.Reader, method string) (string, error) { b, err := ioutil.ReadAll(r) if err != nil { return "", err } switch method { case "md5": return fmt.Sprintf("%x", md5.Sum(b)), nil case "sha1": return fmt.Sprintf("%x", sha1.Sum(b)), nil case "sha256": return fmt.Sprintf("%x", sha256.Sum256(b)), nil case "sha512": return fmt.Sprintf("%x", sha512.Sum512(b)), nil case "adler32": return strconv.FormatUint(uint64(adler32.Checksum(b)), 10), nil case "crc32": return strconv.FormatUint(uint64(crc32.ChecksumIEEE(b)), 10), nil default: return "", fmt.Errorf("hashing method %s is not supported", method) } }
func (z *reader) Reset(r io.Reader, dict []byte) error { if fr, ok := r.(flate.Reader); ok { z.r = fr } else { z.r = bufio.NewReader(r) } _, err := io.ReadFull(z.r, z.scratch[0:2]) if err != nil { return err } h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { return ErrHeader } haveDict := z.scratch[1]&0x20 != 0 if haveDict { _, err = io.ReadFull(z.r, z.scratch[0:4]) if err != nil { return err } checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) if checksum != adler32.Checksum(dict) { return ErrDictionary } } if z.decompressor == nil { if haveDict { z.decompressor = flate.NewReaderDict(z.r, dict) } else { z.decompressor = flate.NewReader(z.r) } } else { z.decompressor.(flate.Resetter).Reset(z.r, dict) } z.digest = adler32.New() return nil }
// Tests by writing up to 4k of data into the checksum func TestEquivalence(t *testing.T) { testarr := make([]byte, 4*1024) sum := 0 for sum != len(testarr) { num, _ := rand.Read(testarr[sum:]) sum += num } for i := 0; i <= len(testarr); i++ { var rolling = New(255) rolling.Write(testarr[:i]) start := 0 if i > 255 { start = i - 255 } expected := adler32.Checksum(testarr[start:i]) actual := rolling.Sum32() if expected != actual { t.Fatalf("%d: expected %x, got %x, %x -> %x", i, expected, actual, testarr[start-1], testarr[i-1]) } } }
func (s *sharder) GetBlockShard(block *pfs.Block) uint64 { return uint64(adler32.Checksum([]byte(block.Hash))) % s.blockModulus }
func (s *sharder) GetShard(file *pfs.File) uint64 { return uint64(adler32.Checksum([]byte(path.Clean(file.Path)))) % s.fileModulus }