func (l *Log) WritePoints(points []models.Point, fields map[string]*tsdb.MeasurementFields, series []*tsdb.SeriesCreate) error {
// add everything to the cache, or return an error if we've hit our max memory
if addedToCache := l.addToCache(points, fields, series, true); !addedToCache {
return fmt.Errorf("WAL backed up flushing to index, hit max memory")
}
// make the write durable if specified
if !l.SkipDurability {
// write the points
pointStrings := make([]string, len(points))
for i, p := range points {
pointStrings[i] = p.String()
}
data := strings.Join(pointStrings, "\n")
compressed := snappy.Encode(nil, []byte(data))
if err := l.writeToLog(pointsEntry, compressed); err != nil {
return err
}
// write the new fields
if len(fields) > 0 {
data, err := json.Marshal(fields)
if err != nil {
return err
}
compressed = snappy.Encode(compressed, data)
if err := l.writeToLog(fieldsEntry, compressed); err != nil {
return err
}
}
// write the new series
if len(series) > 0 {
data, err := json.Marshal(series)
if err != nil {
return err
}
compressed = snappy.Encode(compressed, data)
if err := l.writeToLog(seriesEntry, compressed); err != nil {
return err
}
}
}
// usually skipping the cache is only for testing purposes and this was the easiest
// way to represent the logic (to cache and then immediately flush)
if l.SkipCache {
if err := l.flush(idleFlush); err != nil {
return err
}
}
return nil
}
func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) {
// Compress the buffer if necessary.
var b []byte
if compression == opt.SnappyCompression {
// Allocate scratch enough for compression and block trailer.
if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
w.compressionScratch = make([]byte, n)
}
compressed := snappy.Encode(w.compressionScratch, buf.Bytes())
n := len(compressed)
b = compressed[:n+blockTrailerLen]
b[n] = blockTypeSnappyCompression
} else {
tmp := buf.Alloc(blockTrailerLen)
tmp[0] = blockTypeNoCompression
b = buf.Bytes()
}
// Calculate the checksum.
n := len(b) - 4
checksum := util.NewCRC(b[:n]).Value()
binary.LittleEndian.PutUint32(b[n:], checksum)
// Write the buffer to the file.
_, err = w.writer.Write(b)
if err != nil {
return
}
bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)}
w.offset += uint64(len(b))
return
}
func (p *Protocol) Send(msg proto.Message) error {
if p.w == nil {
return ErrNoWriter
}
data, err := proto.Marshal(msg)
if err != nil {
return err
}
if p.compress {
data = snappy.Encode(nil, data)
}
sizeBuf := make([]byte, 4)
binary.LittleEndian.PutUint32(sizeBuf, uint32(len(data)))
if _, err := p.w.Write(sizeBuf); err != nil {
return err
}
if _, err := p.w.Write(data); err != nil {
return err
}
return nil
}
func SendFrame(w io.Writer, data proto.Message) (err error) {
// marshal response
pbData := []byte{}
if data == nil {
return errors.New("data is nil")
}
pbData, err = proto.Marshal(data)
if err != nil {
return err
}
compressedPb := snappy.Encode(nil, pbData)
err = sendFrame(w, compressedPb)
if err != nil {
return err
}
if data == nil {
return errors.New("data is nil")
}
err = proto.Unmarshal(pbData, data)
return
}
func (e *Engine) writeSeries(series map[string]*tsdb.Series) error {
data, err := json.Marshal(series)
if err != nil {
return err
}
fn := filepath.Join(e.path, SeriesFileExtension+"tmp")
ff, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return err
}
_, err = ff.Write(snappy.Encode(nil, data))
if err != nil {
return err
}
if err := ff.Close(); err != nil {
return err
}
seriesFileName := filepath.Join(e.path, SeriesFileExtension)
if _, err := os.Stat(seriesFileName); !os.IsNotExist(err) {
if err := os.Remove(seriesFileName); err != nil && err != os.ErrNotExist {
return err
}
}
return os.Rename(fn, seriesFileName)
}
func Fuzz(data []byte) int {
n, err := snappy.DecodedLen(data)
if err != nil || n > 1e6 {
return 0
}
if n < 0 {
panic("negative decoded len")
}
dec, err := snappy.Decode(nil, data)
if err != nil {
if dec != nil {
panic("dec is not nil")
}
return 0
}
if len(dec) != n {
println(len(dec), n)
panic("bad decoded len")
}
n = snappy.MaxEncodedLen(len(dec))
enc := snappy.Encode(nil, dec)
if len(enc) > n {
panic("bad encoded len")
}
dec1, err := snappy.Decode(nil, enc)
if err != nil {
panic(err)
}
if bytes.Compare(dec, dec1) != 0 {
panic("not equal")
}
return 1
}
// writeSeriesAndFields will write the compressed fields and series to the meta file. This file persists the data
// in case the server gets shutdown before the WAL has a chance to flush everything to the cache. By default this
// file is flushed on start when bz1 calls LoadMetaDataIndex
func (l *Log) writeSeriesAndFields(fields map[string]*tsdb.MeasurementFields, series []*tsdb.SeriesCreate) error {
if len(fields) == 0 && len(series) == 0 {
return nil
}
sf := &seriesAndFields{Fields: fields, Series: series}
b, err := json.Marshal(sf)
if err != nil {
return err
}
cb := snappy.Encode(nil, b)
l.mu.Lock()
defer l.mu.Unlock()
if _, err := l.metaFile.Write(u64tob(uint64(len(cb)))); err != nil {
return err
}
if _, err := l.metaFile.Write(cb); err != nil {
return err
}
return l.metaFile.Sync()
}
func TestSnappyCompressor(t *testing.T) {
c := SnappyCompressor{}
if c.Name() != "snappy" {
t.Fatalf("expected name to be 'snappy', got %v", c.Name())
}
str := "My Test String"
//Test Encoding
expected := snappy.Encode(nil, []byte(str))
if res, err := c.Encode([]byte(str)); err != nil {
t.Fatalf("failed to encode '%v' with error %v", str, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected encoded value with the result encoded value.")
}
val, err := c.Encode([]byte(str))
if err != nil {
t.Fatalf("failed to encode '%v' with error '%v'", str, err)
}
//Test Decoding
if expected, err := snappy.Decode(nil, val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if res, err := c.Decode(val); err != nil {
t.Fatalf("failed to decode '%v' with error %v", val, err)
} else if bytes.Compare(expected, res) != 0 {
t.Fatal("failed to match the expected decoded value with the result decoded value.")
}
}
// Save stores data to a file.
func (d *Data) Save() error {
// open or create file
fh, err := os.OpenFile(d.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND|os.O_TRUNC, 0655)
if err != nil {
return err
}
defer fh.Close()
// encode content in a buffer
buf := new(bytes.Buffer)
endata := gob.NewEncoder(buf)
endata.Encode(d.Store)
b := buf.Bytes()
// compress content
da := snappy.Encode(nil, b)
// write content to file
_, err = fh.Write(da)
if err != nil {
return err
}
return nil
}
func (b *EncryptBackend) Put(hash string, rawData []byte) (err error) {
// #blobstash/secretbox\n
// data hash\n
// data
var nonce [24]byte
//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
if err := GenerateNonce(&nonce); err != nil {
return err
}
// First we compress the data with snappy
data := snappy.Encode(nil, rawData)
var out bytes.Buffer
out.WriteString("#blobstash/secretbox\n")
out.WriteString(fmt.Sprintf("%v\n", hash))
encData := make([]byte, len(data)+secretbox.Overhead)
secretbox.Seal(encData[0:0], data, &nonce, b.key)
out.Write(nonce[:])
out.Write(encData)
encHash := fmt.Sprintf("%x", blake2b.Sum256(out.Bytes()))
b.dest.Put(encHash, out.Bytes())
b.Lock()
b.index[hash] = encHash
defer b.Unlock()
blobsUploaded.Add(b.dest.String(), 1)
bytesUploaded.Add(b.dest.String(), int64(len(out.Bytes())))
return
}
func (l *WAL) writeToLog(entry WALEntry) (int, error) {
// encode and compress the entry while we're not locked
bytes := make([]byte, defaultBufLen)
b, err := entry.Encode(bytes)
if err != nil {
return -1, err
}
compressed := snappy.Encode(b, b)
l.mu.Lock()
defer l.mu.Unlock()
// Make sure the log has not been closed
select {
case <-l.closing:
return -1, ErrWALClosed
default:
}
// roll the segment file if needed
if err := l.rollSegment(); err != nil {
return -1, fmt.Errorf("error rolling WAL segment: %v", err)
}
// write and sync
if err := l.currentSegmentWriter.Write(entry.Type(), compressed); err != nil {
return -1, fmt.Errorf("error writing WAL entry: %v", err)
}
l.lastWriteTime = time.Now()
return l.currentSegmentID, l.currentSegmentWriter.sync()
}
func (w *WALSegmentWriter) Write(e WALEntry) error {
bytes := make([]byte, defaultBufLen)
b, err := e.Encode(bytes)
if err != nil {
return err
}
compressed := snappy.Encode(b, b)
w.mu.Lock()
defer w.mu.Unlock()
if _, err := w.w.Write([]byte{byte(e.Type())}); err != nil {
return err
}
if _, err = w.w.Write(u32tob(uint32(len(compressed)))); err != nil {
return err
}
if _, err = w.w.Write(compressed); err != nil {
return err
}
// 5 is the 1 byte type + 4 byte uint32 length
w.size += len(compressed) + 5
return nil
}
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
switch fw.CompressionCodec {
case CompressionDeflate:
bb := new(bytes.Buffer)
comp, _ := flate.NewWriter(bb, flate.DefaultCompression)
for block := range toCompress {
_, block.err = comp.Write(block.encoded.Bytes())
block.err = comp.Close()
if block.err == nil {
block.compressed = bb.Bytes()
toWrite <- block
}
bb = new(bytes.Buffer)
comp.Reset(bb)
}
case CompressionNull:
for block := range toCompress {
block.compressed = block.encoded.Bytes()
toWrite <- block
}
case CompressionSnappy:
for block := range toCompress {
block.compressed = snappy.Encode(block.compressed, block.encoded.Bytes())
toWrite <- block
}
}
close(toWrite)
}
func (e *Engine) writeFields(fields map[string]*tsdb.MeasurementFields) error {
// compress and save everything
data, err := json.Marshal(fields)
if err != nil {
return err
}
fn := filepath.Join(e.path, FieldsFileExtension+"tmp")
ff, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return err
}
_, err = ff.Write(snappy.Encode(nil, data))
if err != nil {
return err
}
if err := ff.Close(); err != nil {
return err
}
fieldsFileName := filepath.Join(e.path, FieldsFileExtension)
if _, err := os.Stat(fieldsFileName); !os.IsNotExist(err) {
if err := os.Remove(fieldsFileName); err != nil {
return err
}
}
return os.Rename(fn, fieldsFileName)
}
// Write a record
func (rw *Writer) WriteRecord(data []byte, flags Flags) error {
if rw.Err != nil {
return rw.Err
}
flags = flags | rw.Flags
if flags&NoCompression == 0 {
data = snappy.Encode(rw.compressBuf, data)
}
header := recordHeader{bodyLength: uint32(len(data)), flags: flags}
var headerBuf [recordHeaderStorageSize]byte
header.encode(headerBuf[:])
if size, _ := rw.bytesWriter.Write(headerBuf[:]); size != recordHeaderStorageSize {
return rw.err(ErrWriteBytes)
}
bodyWriter := checksumWriter{writer: rw.bytesWriter, crc: crc32.NewIEEE()}
if size, _ := bodyWriter.Write(data); size != len(data) {
return rw.err(ErrWriteBytes)
}
var checksumBuf [4]byte
binary.LittleEndian.PutUint32(checksumBuf[:], bodyWriter.checksum())
if size, _ := rw.bytesWriter.Write(checksumBuf[:]); size != len(checksumBuf) {
return rw.err(ErrWriteBytes)
}
return nil
}
func (snappyMessageCompressor) compressData(dst, src []byte) (n int, err error) {
p := snappy.Encode(dst, src)
if len(p) > len(dst) {
err = io.ErrShortBuffer
}
n = len(p)
return
}
func (e *Engine) writeSeries(tx *bolt.Tx, series map[string]*tsdb.Series) error {
data, err := json.Marshal(series)
if err != nil {
return err
}
return tx.Bucket([]byte("meta")).Put([]byte("series"), snappy.Encode(nil, data))
}
func (l *Log) writeDeleteEntry(d *deleteData) error {
js, err := json.Marshal(d)
if err != nil {
return err
}
data := snappy.Encode(nil, js)
return l.writeToLog(deleteEntry, data)
}
func (l *internalLevelDBStore) putByKey(key []byte, c Chunk) {
l.concurrentWriteLimit <- struct{}{}
data := snappy.Encode(nil, c.Data())
err := l.db.Put(key, data, nil)
d.Chk.NoError(err)
l.putCount++
l.putBytes += int64(len(data))
<-l.concurrentWriteLimit
}
func (e *Engine) writeFields(tx *bolt.Tx, fields map[string]*tsdb.MeasurementFields) error {
// compress and save everything
data, err := json.Marshal(fields)
if err != nil {
return err
}
return tx.Bucket([]byte("meta")).Put([]byte("fields"), snappy.Encode(nil, data))
}
func block_compressor(in DuplexPipe, block_size, conc_level int) DuplexPipe {
// This is the output of the generator
out := DuplexPipe{make(chan Block, conc_level), make(chan Block, conc_level)}
// Block ready to hold reading
comp_len := snappy.MaxEncodedLen(block_size)
for i := 0; i < conc_level; i++ {
out.Upstream <- Block{make([]byte, comp_len), 0, 0}
}
var comp_buf Block
go func() {
done := make(chan bool, conc_level)
for block := range in.Downstream {
comp_buf = <-out.Upstream
done <- false
go func() {
fmt.Println("Compressing block", block.N, "ID", block.BlockID)
if block.N == block_size {
// We are allocating comp_chunk extra here to know length
// ! Fork snappy to return len(comp_buf.Buf) instead of
// the the actual slice of comp_buf.Buf
comp_chunk := snappy.Encode(comp_buf.Buf, block.Buf)
// this misses the point of having reusable slices... :-(
comp_buf.N = len(comp_chunk)
comp_buf.BlockID = block.BlockID
} else {
comp_buf.N = block.N
comp_buf.BlockID = block.BlockID
copy(comp_buf.Buf[:comp_buf.N], block.Buf)
}
in.Upstream <- block
out.Downstream <- comp_buf
<-done
}()
}
// Wait for them to finish
for i := 0; i < conc_level; i++ {
done <- true
}
close(out.Downstream)
}()
return out
}
// Write will write a compressed block of the points to the current segment file. If the segment
// file is larger than the max size, it will roll over to a new file before performing the write.
// This method will also add the points to the in memory cache
func (p *Partition) Write(points []tsdb.Point) error {
block := make([]byte, 0)
for _, pp := range points {
block = append(block, marshalWALEntry(pp.Key(), pp.UnixNano(), pp.Data())...)
}
b := snappy.Encode(nil, block)
if backoff, ok := func() (time.Duration, bool) {
p.mu.Lock()
defer p.mu.Unlock()
// pause writes for a bit if we've hit the size threshold
if p.memorySize > p.sizeThreshold {
p.backoffCount += 1
return time.Millisecond * 20, true
}
return 0, false
}(); ok {
go p.flushAndCompact(memoryFlush)
time.Sleep(backoff)
}
p.mu.Lock()
defer p.mu.Unlock()
// rotate to a new file if we've gone over our limit
if p.currentSegmentFile == nil || p.currentSegmentSize > p.maxSegmentSize {
err := p.newSegmentFile()
if err != nil {
return err
}
}
if n, err := p.currentSegmentFile.Write(u64tob(uint64(len(b)))); err != nil {
return err
} else if n != 8 {
return fmt.Errorf("expected to write %d bytes but wrote %d", 8, n)
}
if n, err := p.currentSegmentFile.Write(b); err != nil {
return err
} else if n != len(b) {
return fmt.Errorf("expected to write %d bytes but wrote %d", len(b), n)
}
if err := p.currentSegmentFile.Sync(); err != nil {
return err
}
p.currentSegmentSize += int64(8 + len(b))
p.lastWriteTime = time.Now()
for _, pp := range points {
p.addToCache(pp.Key(), pp.Data(), pp.UnixNano())
}
return nil
}
func mustMarshalEntry(entry WALEntry) (WalEntryType, []byte) {
bytes := make([]byte, 1024<<2)
b, err := entry.Encode(bytes)
if err != nil {
panic(fmt.Sprintf("error encoding: %v", err))
}
return entry.Type(), snappy.Encode(b, b)
}
func (l *LevelDBStore) PutMany(chunks []Chunk) (e BackpressureError) {
numBytes := 0
b := new(leveldb.Batch)
for _, c := range chunks {
data := snappy.Encode(nil, c.Data())
numBytes += len(data)
b.Put(l.toChunkKey(c.Hash()), data)
}
l.putBatch(b, numBytes)
return
}
func NewSnappyEncodeOp() stream.Operator {
name := "SnappyEncodeOp"
generator := func() mapper.Worker {
fn := func(obj stream.Object, out mapper.Outputer) error {
compressed := snappy.Encode(nil, obj.([]byte))
out.Sending(1).Send(compressed)
return nil
}
return mapper.NewWorker(fn, name)
}
return mapper.NewClosureOp(generator, nil, name)
}
// writeMessageSet writes a Message Set into w.
// It returns the number of bytes written and any error.
func writeMessageSet(w io.Writer, messages []*Message, compression Compression) (int, error) {
if len(messages) == 0 {
return 0, nil
}
// NOTE(caleb): it doesn't appear to be documented, but I observed that the
// Java client sets the offset of the synthesized message set for a group of
// compressed messages to be the offset of the last message in the set.
compressOffset := messages[len(messages)-1].Offset
switch compression {
case CompressionGzip:
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
if _, err := writeMessageSet(gz, messages, CompressionNone); err != nil {
return 0, err
}
if err := gz.Close(); err != nil {
return 0, err
}
messages = []*Message{
{
Value: buf.Bytes(),
Offset: compressOffset,
},
}
case CompressionSnappy:
var buf bytes.Buffer
if _, err := writeMessageSet(&buf, messages, CompressionNone); err != nil {
return 0, err
}
messages = []*Message{
{
Value: snappy.Encode(nil, buf.Bytes()),
Offset: compressOffset,
},
}
}
enc := NewEncoder(w)
totalSize := 0
for _, message := range messages {
totalSize += 26 + len(message.Key) + len(message.Value)
enc.Encode(int64(message.Offset))
messageSize := 14 + len(message.Key) + len(message.Value)
enc.Encode(int32(messageSize))
enc.Encode(ComputeCrc(message, compression))
enc.Encode(int8(0)) // magic byte
enc.Encode(int8(compression))
enc.Encode(message.Key)
enc.Encode(message.Value)
}
return totalSize, enc.Err()
}
func (l *LevelDBStore) PutMany(chunks []Chunk) (e BackpressureError) {
d.Chk.True(l.internalLevelDBStore != nil, "Cannot use LevelDBStore after Close().")
l.versionSetOnce.Do(l.setVersIfUnset)
numBytes := 0
b := new(leveldb.Batch)
for _, c := range chunks {
data := snappy.Encode(nil, c.Data())
numBytes += len(data)
b.Put(l.toChunkKey(c.Hash()), data)
}
l.putBatch(b, numBytes)
return
}
func (s *hopCipher) encrypt(msg []byte) []byte {
cmsg := make([]byte, snappy.MaxEncodedLen(len(msg)))
cmsg = snappy.Encode(cmsg, msg)
pmsg := PKCS5Padding(cmsg, cipherBlockSize)
buf := make([]byte, len(pmsg)+cipherBlockSize)
iv := buf[:cipherBlockSize]
rand.Read(iv)
encrypter := _cipher.NewCBCEncrypter(s.block, iv)
encrypter.CryptBlocks(buf[cipherBlockSize:], pmsg)
return buf
}
func writeResponse(w io.Writer, id uint64, serr string, response proto.Message) (err error) {
// clean response if error
if serr != "" {
response = nil
}
// marshal response
pbResponse := []byte{}
if response != nil {
pbResponse, err = proto.Marshal(response)
if err != nil {
return err
}
}
// compress serialized proto data
compressedPbResponse := snappy.Encode(nil, pbResponse)
// generate header
header := &wire.ResponseHeader{
Id: proto.Uint64(id),
Error: proto.String(serr),
RawResponseLen: proto.Uint32(uint32(len(pbResponse))),
SnappyCompressedResponseLen: proto.Uint32(uint32(len(compressedPbResponse))),
Checksum: proto.Uint32(crc32.ChecksumIEEE(compressedPbResponse)),
}
// check header size
pbHeader, err := proto.Marshal(header)
if err != err {
return
}
if uint32(len(pbHeader)) > wire.Default_Const_MaxHeaderLen {
return fmt.Errorf("protorpc.writeResponse: header larger than max_header_len: %d.",
len(pbHeader),
)
}
// send header (more)
if err = sendFrame(w, pbHeader); err != nil {
return
}
// send body (end)
if err = sendFrame(w, compressedPbResponse); err != nil {
return
}
return nil
}
func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) {
switch fw.CompressionCodec {
case CompressionNull:
for block := range toCompress {
block.compressed = block.encoded.Bytes()
toWrite <- block
}
case CompressionDeflate:
bb := new(bytes.Buffer)
cw, _ := flate.NewWriter(bb, flate.DefaultCompression)
for block := range toCompress {
bb = new(bytes.Buffer)
cw.Reset(bb)
if _, block.err = cw.Write(block.encoded.Bytes()); block.err != nil {
continue
}
if block.err = cw.Close(); block.err != nil {
continue
}
block.compressed = bb.Bytes()
toWrite <- block
}
case CompressionSnappy:
var bb *bytes.Buffer
var dst []byte
for block := range toCompress {
checksum := crc32.ChecksumIEEE(block.encoded.Bytes())
dst = snappy.Encode(nil, block.encoded.Bytes())
bb = bytes.NewBuffer(dst)
if block.err = binary.Write(bb, binary.BigEndian, checksum); block.err != nil {
continue
}
block.compressed = bb.Bytes()
toWrite <- block
}
}
close(toWrite)
}