Exemplo n.º 1
0
func TestMultiCoreChecksummedWriter(t *testing.T) {
	buf := &bytes.Buffer{}
	cw := NewMultiCoreChecksummedWriter(buf, 16, crc32.NewIEEE, runtime.GOMAXPROCS(0))
	if cw == nil {
		t.Fatal(cw)
	}
	n, err := cw.Write([]byte("12345678901234567890"))
	if n != 20 {
		t.Fatal(n)
	}
	if err != nil {
		t.Fatal(err)
	}
	hash := crc32.NewIEEE()
	hash.Write([]byte("1234567890123456"))
	n, err = cw.Write([]byte("ghijklmnopqrstuvwxyz"))
	if n != 20 {
		t.Fatal(n)
	}
	if err != nil {
		t.Fatal(err)
	}
	hash2 := crc32.NewIEEE()
	hash2.Write([]byte("7890ghijklmnopqr"))
	err = cw.Close()
	if err != nil {
		t.Fatal(err)
	}
	hash3 := crc32.NewIEEE()
	hash3.Write([]byte("stuvwxyz"))
	if !bytes.Equal(buf.Bytes(), []byte("1234567890123456"+string(hash.Sum(nil))+"7890ghijklmnopqr"+string(hash2.Sum(nil))+"stuvwxyz"+string(hash3.Sum(nil)))) {
		t.Fatalf("%#v", string(buf.Bytes()))
	}
}
Exemplo n.º 2
0
func makeOuts(testIntents []*intents.Intent, demux *Demultiplexer, outChecksum map[string]hash.Hash, demuxOuts map[string]*RegularCollectionReceiver, outLengths map[string]*int, errCh chan<- error) {
	for _, dbc := range testIntents {
		ns := dbc.Namespace()
		sum := crc32.NewIEEE()
		muxOut := &RegularCollectionReceiver{
			Intent: dbc,
			Demux:  demux,
			Origin: ns,
		}
		outLength := 0

		outChecksum[ns] = sum
		demuxOuts[ns] = muxOut
		outLengths[ns] = &outLength

		demuxOuts[ns].Open()
		go func() {
			bs := make([]byte, db.MaxBSONSize)
			var err error
			for {
				var length int
				length, err = muxOut.Read(bs)
				if err != nil {
					break
				}
				sum.Write(bs[:length])
				outLength += len(bs[:length])
			}
			if err == io.EOF {
				err = nil
			}
			errCh <- err
		}()
	}
}
Exemplo n.º 3
0
func (m *Dot11) ChecksumValid() bool {
	// only for CTRL and MGMT frames
	h := crc32.NewIEEE()
	h.Write(m.Contents)
	h.Write(m.Payload)
	return m.Checksum == h.Sum32()
}
Exemplo n.º 4
0
func (d *decoder) run() error {
	// init hash
	d.crcHash = crc32.NewIEEE()
	// for each part
	for {
		// create a part
		d.part = new(Part)
		// read the header
		if err := d.readHeader(); err != nil {
			return err
		}
		// read part header if available
		if d.multipart {
			if err := d.readPartHeader(); err != nil {
				return err
			}
		}
		// decode the part body
		if err := d.readBody(); err != nil {
			return err
		}
		// add part to list
		d.parts = append(d.parts, d.part)
		// validate part
		if err := d.part.validate(); err != nil {
			return err
		}
	}
	return nil
}
Exemplo n.º 5
0
// Open returns a ReadCloser that provides access to the File's contents.
// Multiple files may be read concurrently.
func (f *File) Open() (rc io.ReadCloser, err error) {
	bodyOffset, err := f.findBodyOffset()
	if err != nil {
		return
	}
	size := int64(f.CompressedSize64)
	r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
	switch f.Method {
	case Store: // (no compression)
		rc = ioutil.NopCloser(r)
	case Deflate:
		rc, err = zlib.NewReader(r)
		if err != nil {
			return
		}
	default:
		err = ErrAlgorithm
		return
	}
	var desr io.Reader
	if f.hasDataDescriptor() {
		desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen)
	}
	rc = &checksumReader{rc, crc32.NewIEEE(), f, desr, nil}
	return
}
Exemplo n.º 6
0
// handleIndex returns the index XML file to the client.
func (s *EC2HTTPTestStorage) handleIndex(w http.ResponseWriter, req *http.Request) {
	lbr := &listBucketResult{
		Name:        "juju-dist",
		Prefix:      "",
		Marker:      "",
		MaxKeys:     1000,
		IsTruncated: false,
	}
	names := []string{}
	for name := range s.files {
		names = append(names, name)
	}
	sort.Strings(names)
	for _, name := range names {
		h := crc32.NewIEEE()
		h.Write([]byte(s.files[name]))
		contents := &contents{
			Key:          name,
			LastModified: time.Now(),
			ETag:         fmt.Sprintf("%x", h.Sum(nil)),
			Size:         len([]byte(s.files[name])),
			StorageClass: "STANDARD",
		}
		lbr.Contents = append(lbr.Contents, contents)
	}
	buf, err := xml.Marshal(lbr)
	if err != nil {
		http.Error(w, fmt.Sprintf("500 %v", err), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/xml")
	w.Write(buf)
}
Exemplo n.º 7
0
Arquivo: log.go Projeto: 0x434D53/raft
// decode deserializes one log entry from the passed io.Reader.
func (e *logEntry) decode(r io.Reader) error {
	header := make([]byte, 24)

	if _, err := r.Read(header); err != nil {
		return err
	}

	command := make([]byte, binary.LittleEndian.Uint32(header[20:24]))

	if _, err := r.Read(command); err != nil {
		return err
	}

	crc := binary.LittleEndian.Uint32(header[:4])

	check := crc32.NewIEEE()
	check.Write(header[4:])
	check.Write(command)

	if crc != check.Sum32() {
		return errInvalidChecksum
	}

	e.Term = binary.LittleEndian.Uint64(header[4:12])
	e.Index = binary.LittleEndian.Uint64(header[12:20])
	e.Command = command

	return nil
}
Exemplo n.º 8
0
// UnmarshalBinary reads header from the provided data slice.
func (h *header) UnmarshalBinary(data []byte) error {
	// header length
	if len(data) != HeaderLen {
		return errors.New("xz: wrong file header length")
	}

	// magic header
	if !bytes.Equal(headerMagic, data[:6]) {
		return errHeaderMagic
	}

	// checksum
	crc := crc32.NewIEEE()
	crc.Write(data[6:8])
	if uint32LE(data[8:]) != crc.Sum32() {
		return errors.New("xz: invalid checksum for file header")
	}

	// stream flags
	if data[6] != 0 {
		return errInvalidFlags
	}
	flags := data[7]
	if err := verifyFlags(flags); err != nil {
		return err
	}

	h.flags = flags
	return nil
}
Exemplo n.º 9
0
// MarshalBinary converts footer values into an xz file footer. Note
// that the footer value is checked for correctness.
func (f *footer) MarshalBinary() (data []byte, err error) {
	if err = verifyFlags(f.flags); err != nil {
		return nil, err
	}
	if !(minIndexSize <= f.indexSize && f.indexSize <= maxIndexSize) {
		return nil, errors.New("xz: index size out of range")
	}
	if f.indexSize%4 != 0 {
		return nil, errors.New(
			"xz: index size not aligned to four bytes")
	}

	data = make([]byte, footerLen)

	// backward size (index size)
	s := (f.indexSize / 4) - 1
	putUint32LE(data[4:], uint32(s))
	// flags
	data[9] = f.flags
	// footer magic
	copy(data[10:], footerMagic)

	// CRC-32
	crc := crc32.NewIEEE()
	crc.Write(data[4:10])
	putUint32LE(data, crc.Sum32())

	return data, nil
}
Exemplo n.º 10
0
func SmallIndexer(gene genome.Gene) (index []byte) {
	hash := crc32.NewIEEE()
	// 	//TODO: fix me
	hash.Write(gene)
	index = hash.Sum()
	return
}
Exemplo n.º 11
0
// Used internally to create and initialize a new request.
func newRequest(request *http.Request, conn net.Conn, startTime time.Time) *Request {
	fReq := new(Request)
	fReq.Context = make(map[string]interface{})
	fReq.HttpRequest = request
	fReq.StartTime = startTime
	fReq.connection = conn
	if conn != nil {
		fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr)
	}

	// create a semi-unique id to track a connection in the logs
	// ID is the least significant decimal digits of time with some randomization
	// the last 3 zeros of time.Nanoseconds appear to always be zero
	var ut = fReq.StartTime.UnixNano()
	fReq.ID = fmt.Sprintf("%010x", (ut-(ut-(ut%1e12)))+int64(rand.Intn(999)))
	fReq.PipelineStageStats = list.New()
	fReq.pipelineHash = crc32.NewIEEE()

	// Support for 100-continue requests
	// http.Server (and presumably google app engine) already handle this
	// case.  So we don't need to do anything if we don't own the
	// connection.
	if conn != nil && request.Header.Get("Expect") == "100-continue" {
		request.Body = &continueReader{req: fReq, r: request.Body}
	}

	return fReq
}
Exemplo n.º 12
0
func TestFileBigWriteWeirdBlockSize(t *testing.T) {
	client := getClient(t)

	baleet(t, "/_test/create/4.txt")
	mkdirp(t, "/_test/create")
	writer, err := client.CreateFile("/_test/create/4.txt", 1, 1050000, 0755)
	require.NoError(t, err)

	mobydick, err := os.Open("test/mobydick.txt")
	require.NoError(t, err)

	n, err := io.Copy(writer, mobydick)
	require.NoError(t, err)
	assert.EqualValues(t, 1257276, n)

	err = writer.Close()
	require.NoError(t, err)

	reader, err := client.Open("/_test/create/4.txt")
	require.NoError(t, err)

	hash := crc32.NewIEEE()
	n, err = io.Copy(hash, reader)
	assert.Nil(t, err)
	assert.EqualValues(t, 1257276, n)
	assert.EqualValues(t, 0x199d1ae6, hash.Sum32())
}
Exemplo n.º 13
0
func ProcessBuffer(c <-chan m.MetricDefinition) {
	buf := make(map[uint32][]m.MetricDefinition)

	// flush buffer 10 times every second
	t := time.NewTicker(time.Millisecond * 100)
	for {
		select {
		case b := <-c:
			if b.OrgId != 0 {
				//get hash.
				h := crc32.NewIEEE()
				h.Write([]byte(b.Name))
				hash := h.Sum32() % uint32(1024)
				if _, ok := buf[hash]; !ok {
					buf[hash] = make([]m.MetricDefinition, 0)
				}
				buf[hash] = append(buf[hash], b)
			}
		case <-t.C:
			//copy contents of buffer
			for hash, metrics := range buf {
				currentBuf := make([]m.MetricDefinition, len(metrics))
				copy(currentBuf, metrics)
				delete(buf, hash)
				//log.Info(fmt.Sprintf("flushing %d items in buffer now", len(currentBuf)))
				msgString, err := json.Marshal(currentBuf)
				if err != nil {
					log.Error(0, "Failed to marshal metrics payload.", err)
				} else {
					go Publish(fmt.Sprintf("%d", hash), msgString)
				}
			}
		}
	}
}
Exemplo n.º 14
0
Arquivo: reader.go Projeto: rboyer/wal
func newReader(fn string) (*reader, error) {
	f, err := os.OpenFile(fn, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	var data gommap.MMap
	var zero bool
	if fi.Size() <= 0 {
		zero = true
	} else {
		data, err = gommap.Map(f.Fd(), gommap.PROT_READ, gommap.MAP_SHARED)
		if err != nil {
			return nil, err
		}
	}

	r := &reader{
		cs:   crc32.NewIEEE(),
		data: data,
		zero: zero,
		done: zero, // noop reader if zero-byte file
	}
	r.bv.Data = data

	return r, nil
}
Exemplo n.º 15
0
func checksumsPass(romname string, rom *clrmamepro.Block, f io.Reader) bool {
	crc := crc32.NewIEEE()
	md := md5.New()
	sha := sha1.New()
	checksums := io.MultiWriter(crc, md, sha)

	_, err := io.Copy(checksums, f)
	if err != nil {
		die("checksumming ROM %q: %v", romname, err)
	}

	compare := func(h hash.Hash, expected string) bool {
		return strings.ToUpper(fmt.Sprintf("%x", h.Sum(nil))) == strings.ToUpper(expected)
	}
	if !compare(crc, rom.Texts[fCRC32]) {
		alert("BAD CRC32", romname)
		return false
	}
	if !compare(md, rom.Texts[fMD5]) {
		alert("BAD MD5", romname)
		return false
	}
	if !compare(sha, rom.Texts[fSHA1]) {
		alert("BAD SHA1", romname)
		return false
	}

	return true
}
Exemplo n.º 16
0
// UnmarshalBinary sets the footer value by unmarshalling an xz file
// footer.
func (f *footer) UnmarshalBinary(data []byte) error {
	if len(data) != footerLen {
		return errors.New("xz: wrong footer length")
	}

	// magic bytes
	if !bytes.Equal(data[10:], footerMagic) {
		return errors.New("xz: footer magic invalid")
	}

	// CRC-32
	crc := crc32.NewIEEE()
	crc.Write(data[4:10])
	if uint32LE(data) != crc.Sum32() {
		return errors.New("xz: footer checksum error")
	}

	var g footer
	// backward size (index size)
	g.indexSize = (int64(uint32LE(data[4:])) + 1) * 4

	// flags
	if data[8] != 0 {
		return errInvalidFlags
	}
	g.flags = data[9]
	if err := verifyFlags(g.flags); err != nil {
		return err
	}

	*f = g
	return nil
}
Exemplo n.º 17
0
func BenchmarkEncoding(b *testing.B) {
	h := crc32.NewIEEE()
	k := 1
	input := make([]byte, b.N)
	b.ResetTimer()
	_, _ = Encode(k, h, input, len(input)*2)
}
Exemplo n.º 18
0
// Test a TxRecord with some trailing bytes
func TestTrailingBytesTxRecord(t *testing.T) {
	buf, _, err := genTxValue(0xfa, []byte{0xff, 0xff, 0xff})
	// Add some trailing bytes to the tx record
	buf = append(buf, byte(0xff))
	buf = append(buf, byte(0xff))
	buf = append(buf, byte(0xff))

	summer := crc32.NewIEEE()
	_, err = summer.Write(buf)
	if err != nil {
		t.Error(err)
	}
	crc32sum := summer.Sum32()

	r, err := genTestCaseHeader(buf, 1, crc32sum)
	if err != nil {
		t.Error(err)
	}

	walker, err := NewReaderWalker(r)
	if err != nil {
		t.Error(err)
	}

	_, err = walker.Next()
	if err != ErrRemainingBytesForRecord {
		t.Error(err)
	}

	_, err = walker.Next()
	if err != io.EOF {
		t.Errorf("expected EOF, got %v", err)
	}
}
Exemplo n.º 19
0
// Begin a transaction
func (log *Log) BeginTx() *LogTx {
	tx := &LogTx{}
	tx.log = log
	tx.buf = new(bytes.Buffer)
	tx.crc = crc32.NewIEEE()
	return tx
}
Exemplo n.º 20
0
// Generate a byet slice representing an entry in a Tx record
func genTxValue(kind uint16, val []byte) (chunk []byte, crc32sum uint32, err error) {
	buf := new(bytes.Buffer)

	err = binary.Write(buf, binary.LittleEndian, kind)
	if err != nil {
		return nil, 0, err
	}

	err = binary.Write(buf, binary.LittleEndian, uint16(len(val)))
	if err != nil {
		return nil, 0, err
	}

	_, err = buf.Write(val)
	if err != nil {
		return nil, 0, err
	}

	summer := crc32.NewIEEE()
	_, err = summer.Write(val)
	if err != nil {
		return nil, 0, err
	}

	return buf.Bytes(), summer.Sum32(), nil
}
Exemplo n.º 21
0
func (d *decoder) readBody() error {
	// ready the part body
	d.part.Body = make([]byte, 0)
	// reset special
	d.awaitingSpecial = false
	// setup crc hash
	d.part.crcHash = crc32.NewIEEE()
	// each line
	for {
		line, err := d.buf.ReadBytes('\n')
		if err != nil {
			return err
		}
		// strip linefeeds (some use CRLF some LF)
		line = bytes.TrimRight(line, "\r\n")
		// check for =yend
		if len(line) >= 5 && string(line[:5]) == "=yend" {
			return d.parseTrailer(string(line))
		}
		// decode
		b := d.decode(line)
		// update hashs
		d.part.crcHash.Write(b)
		d.crcHash.Write(b)
		// decode
		d.part.Body = append(d.part.Body, b...)
	}
	return nil
}
Exemplo n.º 22
0
func (e *Engine) crc32() error {
	data, err := computeHash(crc32.NewIEEE(), e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}
Exemplo n.º 23
0
func (e *encoder) writeChunk(b []byte, name string) {
	if e.err != nil {
		return
	}
	n := uint32(len(b))
	if int(n) != len(b) {
		e.err = UnsupportedError(name + " chunk is too large: " + strconv.Itoa(len(b)))
		return
	}
	writeUint32(e.header[0:4], n)
	e.header[4] = name[0]
	e.header[5] = name[1]
	e.header[6] = name[2]
	e.header[7] = name[3]
	crc := crc32.NewIEEE()
	crc.Write(e.header[4:8])
	crc.Write(b)
	writeUint32(e.footer[0:4], crc.Sum32())

	_, e.err = e.w.Write(e.header[0:8])
	if e.err != nil {
		return
	}
	_, e.err = e.w.Write(b)
	if e.err != nil {
		return
	}
	_, e.err = e.w.Write(e.footer[0:4])
}
Exemplo n.º 24
0
func NewArticle(p []byte, data *ArticleData, subject string) *Article {
	buf := new(bytes.Buffer)
	buf.WriteString(fmt.Sprintf("From: %s\r\n", Config.Global.From))
	buf.WriteString(fmt.Sprintf("Newsgroups: %s\r\n", Config.Global.DefaultGroup))
	buf.WriteString(fmt.Sprintf("Message-ID: <%d$gps@gopoststuff>\r\n", time.Now().UnixNano()))
	// art.headers['Message-ID'] = '<%.5f.%d@%s>' % (time.time(), partnum, self.conf['server']['hostname'])
	//headers["X-Newsposter"] = "gopoststuff alpha - https://github.com/madcowfred/gopoststuff"
	buf.WriteString(fmt.Sprintf("X-Newsposter: gopoststuff alpha - https://github.com/madcowfred/gopoststuff\r\n"))

	// Build subject
	// spec: c1 [fnum/ftotal] - "filename" yEnc (pnum/ptotal)
	buf.WriteString(fmt.Sprintf("Subject: %s [%d/%d] - \"%s\" yEnc (%d/%d)\r\n\r\n", subject, data.FileNum, data.FileTotal, data.FileName, data.PartNum, data.PartTotal))

	// yEnc begin line
	buf.WriteString(fmt.Sprintf("=ybegin part=%d total=%d line=128 size=%d name=%s\r\n", data.PartNum, data.PartTotal, data.FileSize, data.FileName))
	// yEnc part line
	buf.WriteString(fmt.Sprintf("=ypart begin=%d end=%d\r\n", data.PartBegin+1, data.PartEnd))

	//log.Debug("%+v", buf)
	// Encoded data
	yencode.Encode(p, buf)
	// yEnc end line
	h := crc32.NewIEEE()
	h.Write(p)
	buf.WriteString(fmt.Sprintf("=yend size=%d part=%d pcrc32=%08X\r\n", data.PartSize, data.PartNum, h.Sum32()))

	return &Article{Body: buf.Bytes()}
}
Exemplo n.º 25
0
// Open returns a ReadCloser that provides access to the File's contents.
func (f *File) Open() (rc io.ReadCloser, err os.Error) {
	off := int64(f.headerOffset)
	size := int64(f.CompressedSize)
	if f.bodyOffset == 0 {
		r := io.NewSectionReader(f.zipr, off, f.zipsize-off)
		if err = readFileHeader(f, r); err != nil {
			return
		}
		if f.bodyOffset, err = r.Seek(0, os.SEEK_CUR); err != nil {
			return
		}
		if size == 0 {
			size = int64(f.CompressedSize)
		}
	}
	if f.hasDataDescriptor() && size == 0 {
		// permit SectionReader to see the rest of the file
		size = f.zipsize - (off + f.bodyOffset)
	}
	r := io.NewSectionReader(f.zipr, off+f.bodyOffset, size)
	switch f.Method {
	case Store: // (no compression)
		rc = ioutil.NopCloser(r)
	case Deflate:
		rc = flate.NewReader(r)
	default:
		err = UnsupportedMethod
	}
	if rc != nil {
		rc = &checksumReader{rc, crc32.NewIEEE(), f, r}
	}
	return
}
Exemplo n.º 26
0
func NewChecksumIndexOutput(main IndexOutput) *ChecksumIndexOutput {
	return &ChecksumIndexOutput{
		IndexOutputImpl: NewIndexOutput(main),
		main:            main,
		digest:          crc32.NewIEEE(),
	}
}
Exemplo n.º 27
0
Arquivo: reader.go Projeto: gmwu/go
// Open方法返回一个io.ReadCloser接口,提供读取文件内容的方法。
// 可以同时读取多个文件。
func (f *File) Open() (rc io.ReadCloser, err error) {
	bodyOffset, err := f.findBodyOffset()
	if err != nil {
		return
	}
	size := int64(f.CompressedSize64)
	r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
	dcomp := decompressor(f.Method)
	if dcomp == nil {
		err = ErrAlgorithm
		return
	}
	rc = dcomp(r)
	var desr io.Reader
	if f.hasDataDescriptor() {
		desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen)
	}
	rc = &checksumReader{
		rc:   rc,
		hash: crc32.NewIEEE(),
		f:    f,
		desr: desr,
	}
	return
}
Exemplo n.º 28
0
// CreateHeader adds a file to the zip file using the provided FileHeader
// for the file metadata.
// It returns a Writer to which the file contents should be written.
//
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close. The provided FileHeader fh
// must not be modified after a call to CreateHeader.
func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
	if w.last != nil && !w.last.closed {
		if err := w.last.close(); err != nil {
			return nil, err
		}
	}
	if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh {
		// See https://golang.org/issue/11144 confusion.
		return nil, errors.New("archive/zip: invalid duplicate FileHeader")
	}

	fh.Flags |= 0x8 // we will write a data descriptor
	// TODO(alex): Look at spec and see if these need to be changed
	// when using encryption.
	fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
	fh.ReaderVersion = zipVersion20

	fw := &fileWriter{
		zipw:      w.cw,
		compCount: &countWriter{w: w.cw},
		crc32:     crc32.NewIEEE(),
	}
	// Get the compressor before possibly changing Method to 99 due to password
	comp := compressor(fh.Method)
	if comp == nil {
		return nil, ErrAlgorithm
	}
	// check for password
	var sw io.Writer = fw.compCount
	if fh.password != nil {
		// we have a password and need to encrypt.
		fh.writeWinZipExtra()
		fh.Method = 99 // ok to change, we've gotten the comp and wrote extra
		ew, err := newEncryptionWriter(sw, fh.password, fw)
		if err != nil {
			return nil, err
		}
		sw = ew
	}
	var err error
	fw.comp, err = comp(sw)
	if err != nil {
		return nil, err
	}
	fw.rawCount = &countWriter{w: fw.comp}

	h := &header{
		FileHeader: fh,
		offset:     uint64(w.cw.count),
	}
	w.dir = append(w.dir, h)
	fw.header = h

	if err := writeHeader(w.cw, fh); err != nil {
		return nil, err
	}

	w.last = fw
	return fw, nil
}
Exemplo n.º 29
0
// Write a record
func (rw *Writer) WriteRecord(data []byte, flags Flags) error {
	if rw.Err != nil {
		return rw.Err
	}
	flags = flags | rw.Flags
	if flags&NoCompression == 0 {
		data = snappy.Encode(rw.compressBuf, data)
	}
	header := recordHeader{bodyLength: uint32(len(data)), flags: flags}
	var headerBuf [recordHeaderStorageSize]byte
	header.encode(headerBuf[:])
	if size, _ := rw.bytesWriter.Write(headerBuf[:]); size != recordHeaderStorageSize {
		return rw.err(ErrWriteBytes)
	}
	bodyWriter := checksumWriter{writer: rw.bytesWriter, crc: crc32.NewIEEE()}
	if size, _ := bodyWriter.Write(data); size != len(data) {
		return rw.err(ErrWriteBytes)
	}
	var checksumBuf [4]byte
	binary.LittleEndian.PutUint32(checksumBuf[:], bodyWriter.checksum())
	if size, _ := rw.bytesWriter.Write(checksumBuf[:]); size != len(checksumBuf) {
		return rw.err(ErrWriteBytes)
	}
	return nil
}
Exemplo n.º 30
0
func makeIns(testIntents []*intents.Intent, mux *Multiplexer, inChecksum map[string]hash.Hash, muxIns map[string]*MuxIn, inLengths map[string]*int, errCh chan<- error) {
	for index, dbc := range testIntents {
		ns := dbc.Namespace()
		sum := crc32.NewIEEE()
		muxIn := &MuxIn{Intent: dbc, Mux: mux}
		inLength := 0

		inChecksum[ns] = sum
		muxIns[ns] = muxIn
		inLengths[ns] = &inLength

		go func(index int) {
			err := muxIn.Open()
			if err != nil {
				errCh <- err
				return
			}
			staticBSONBuf := make([]byte, db.MaxBSONSize)
			for i := 0; i < 10000; i++ {
				bsonBytes, _ := bson.Marshal(testDoc{Bar: index * i, Baz: ns})
				bsonBuf := staticBSONBuf[:len(bsonBytes)]
				copy(bsonBuf, bsonBytes)
				muxIn.Write(bsonBuf)
				sum.Write(bsonBuf)
				inLength += len(bsonBuf)
			}
			err = muxIn.Close()
			errCh <- err
		}(index)
	}
}