Esempio n. 1
0
// readAll read entire referenced block.
func (p *bInfo) readAll(r io.ReaderAt, checksum bool) (b []byte, err error) {
	raw := make([]byte, p.size+5)
	_, err = readFullAt(r, raw, int64(p.offset))
	if err != nil {
		return
	}

	crcb := raw[len(raw)-4:]
	raw = raw[:len(raw)-4]

	if checksum {
		sum := binary.LittleEndian.Uint32(crcb)
		sum = hash.UnmaskCRC32(sum)
		crc := hash.NewCRC32C()
		crc.Write(raw)
		if crc.Sum32() != sum {
			err = errors.ErrCorrupt("block checksum mismatch")
			return
		}
	}

	compression := raw[len(raw)-1]
	b = raw[:len(raw)-1]

	switch compression {
	case kNoCompression:
	case kSnappyCompression:
		return snappy.Decode(nil, b)
	default:
		err = errors.ErrCorrupt("bad block type")
	}

	return
}
Esempio n. 2
0
func (w *Writer) write(rtype uint, record []byte) (err error) {
	rlen := len(record)
	buf := &w.buf
	buf.Reset()

	crc := hash.NewCRC32C()
	crc.Write([]byte{byte(rtype)})
	crc.Write(record)
	binary.Write(buf, binary.LittleEndian, hash.MaskCRC32(crc.Sum32()))

	buf.WriteByte(byte(rlen & 0xff))
	buf.WriteByte(byte(rlen >> 8))
	buf.WriteByte(byte(rtype))

	_, err = buf.WriteTo(w.w)
	if err == nil {
		_, err = w.w.Write(record)
	}
	return err
}
Esempio n. 3
0
func (t *Writer) write(buf []byte, bi *bInfo, raw bool) (err error) {
	compression := opt.NoCompression
	if !raw {
		compression = t.o.GetCompressionType()
	}
	switch compression {
	case opt.SnappyCompression:
		buf, err = snappy.Encode(nil, buf)
		if err != nil {
			return
		}
	}

	if bi != nil {
		bi.offset = uint64(t.off)
		bi.size = uint64(len(buf))
	}

	_, err = t.w.Write(buf)
	if err != nil {
		return
	}

	compbit := []byte{byte(compression)}
	_, err = t.w.Write(compbit)
	if err != nil {
		return
	}

	crc := hash.NewCRC32C()
	crc.Write(buf)
	crc.Write(compbit)
	err = binary.Write(t.w, binary.LittleEndian, hash.MaskCRC32(crc.Sum32()))
	if err != nil {
		return
	}

	t.off += len(buf) + 5
	return
}
Esempio n. 4
0
func (r *Reader) read() (ret []byte, rtype uint, err error) {
retry:
	if len(r.buf) < kHeaderSize {
		if r.eof {
			if len(r.buf) > 0 {
				r.drop(len(r.buf), "truncated record at end of file")
				r.rbuf = nil
				r.buf = nil
			}
			rtype = tEof
			return
		}

		if r.rbuf == nil {
			r.rbuf = make([]byte, BlockSize)
		} else {
			r.off++
		}

		var n int
		n, err = r.r.ReadAt(r.rbuf, int64(r.off)*BlockSize)
		if err != nil {
			if err == io.EOF {
				err = nil
			} else {
				return
			}
		}
		r.buf = r.rbuf[:n]
		if n < BlockSize {
			r.eof = true
			goto retry
		}
	}

	// decode record length and type
	recLen := int(r.buf[4]) | (int(r.buf[5]) << 8)
	rtype = uint(r.buf[6])

	// check whether the header is sane
	if len(r.buf) < kHeaderSize+recLen || rtype > tLast {
		rtype = tCorrupt
		r.drop(len(r.buf), "header corrupted")
	} else if r.checksum {
		// decode the checksum
		recCrc := hash.UnmaskCRC32(binary.LittleEndian.Uint32(r.buf))
		crc := hash.NewCRC32C()
		crc.Write(r.buf[6 : kHeaderSize+recLen])
		if crc.Sum32() != recCrc {
			// Drop the rest of the buffer since "length" itself may have
			// been corrupted and if we trust it, we could find some
			// fragment of a real journal record that just happens to look
			// like a valid journal record.
			rtype = tCorrupt
			r.drop(len(r.buf), "checksum mismatch")
		}
	}

	if rtype == tCorrupt {
		// report bytes drop
		r.buf = nil
	} else {
		ret = r.buf[kHeaderSize : kHeaderSize+recLen]
		r.buf = r.buf[kHeaderSize+recLen:]
	}

	return
}