Example #1
0
func (e *encoder) encode(rec *walpb.Record) error {
	e.crc.Write(rec.Data)
	rec.Crc = e.crc.Sum32()
	data, err := rec.Marshal()
	if err != nil {
		return err
	}
	if err := writeInt64(e.bw, int64(len(data))); err != nil {
		return err
	}
	_, err = e.bw.Write(data)
	return err
}
Example #2
0
func (d *decoder) decodeRecord(rec *walpb.Record) error {
	if len(d.brs) == 0 {
		return io.EOF
	}

	l, err := readInt64(d.brs[0])
	if err == io.EOF || (err == nil && l == 0) {
		// hit end of file or preallocated space
		d.brs = d.brs[1:]
		if len(d.brs) == 0 {
			return io.EOF
		}
		d.lastValidOff = 0
		return d.decodeRecord(rec)
	}
	if err != nil {
		return err
	}

	recBytes, padBytes := decodeFrameSize(l)

	data := make([]byte, recBytes+padBytes)
	if _, err = io.ReadFull(d.brs[0], data); err != nil {
		// ReadFull returns io.EOF only if no bytes were read
		// the decoder should treat this as an ErrUnexpectedEOF instead.
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return err
	}
	if err := rec.Unmarshal(data[:recBytes]); err != nil {
		if d.isTornEntry(data) {
			return io.ErrUnexpectedEOF
		}
		return err
	}

	// skip crc checking if the record type is crcType
	if rec.Type != crcType {
		d.crc.Write(rec.Data)
		if err := rec.Validate(d.crc.Sum32()); err != nil {
			if d.isTornEntry(data) {
				return io.ErrUnexpectedEOF
			}
			return err
		}
	}
	// record decoded as valid; point last valid offset to end of record
	d.lastValidOff += recBytes + padBytes + 8
	return nil
}
Example #3
0
func (e *encoder) encode(rec *walpb.Record) error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.crc.Write(rec.Data)
	rec.Crc = e.crc.Sum32()
	var (
		data []byte
		err  error
		n    int
	)

	if rec.Size() > len(e.buf) {
		data, err = rec.Marshal()
		if err != nil {
			return err
		}
	} else {
		n, err = rec.MarshalTo(e.buf)
		if err != nil {
			return err
		}
		data = e.buf[:n]
	}
	if err := writeInt64(e.bw, int64(len(data)), e.uint64buf); err != nil {
		return err
	}
	_, err = e.bw.Write(data)
	return err
}
Example #4
0
func (d *decoder) decode(rec *walpb.Record) error {
	d.mu.Lock()
	defer d.mu.Unlock()

	rec.Reset()
	l, err := readInt64(d.br)
	if err != nil {
		return err
	}
	data := make([]byte, l)
	if _, err = io.ReadFull(d.br, data); err != nil {
		// ReadFull returns io.EOF only if no bytes were read
		// the decoder should treat this as an ErrUnexpectedEOF instead.
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return err
	}
	if err := rec.Unmarshal(data); err != nil {
		return err
	}
	// skip crc checking if the record type is crcType
	if rec.Type == crcType {
		return nil
	}
	d.crc.Write(rec.Data)
	return rec.Validate(d.crc.Sum32())
}
Example #5
0
func (d *decoder) decode(rec *walpb.Record) error {
	rec.Reset()
	l, err := readInt64(d.br)
	if err != nil {
		return err
	}
	data := make([]byte, l)
	if _, err = io.ReadFull(d.br, data); err != nil {
		return err
	}
	if err := rec.Unmarshal(data); err != nil {
		return err
	}
	// skip crc checking if the record type is crcType
	if rec.Type == crcType {
		return nil
	}
	d.crc.Write(rec.Data)
	return rec.Validate(d.crc.Sum32())
}
Example #6
0
func (e *encoder) encode(rec *walpb.Record) error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.crc.Write(rec.Data)
	rec.Crc = e.crc.Sum32()
	var (
		data []byte
		err  error
		n    int
	)

	if rec.Size() > len(e.buf) {
		data, err = rec.Marshal()
		if err != nil {
			return err
		}
	} else {
		n, err = rec.MarshalTo(e.buf)
		if err != nil {
			return err
		}
		data = e.buf[:n]
	}

	lenField, padBytes := encodeFrameSize(len(data))
	if err = writeUint64(e.bw, lenField, e.uint64buf); err != nil {
		return err
	}

	if padBytes != 0 {
		data = append(data, make([]byte, padBytes)...)
	}
	_, err = e.bw.Write(data)
	return err
}
Example #7
0
func (d *decoder) decode(rec *walpb.Record) error {
	rec.Reset()
	d.mu.Lock()
	defer d.mu.Unlock()
	return d.decodeRecord(rec)
}