func (block *Block) Flush() error {
block.newStreamsLock.Lock()
defer block.newStreamsLock.Unlock()
if len(block.NewStreams) == 0 {
return nil
}
block.logLock.Lock()
defer block.logLock.Unlock()
// There are streams that need to be flushed to disk
file, err := protofile.Write(block.logFilename())
if err != nil {
return err
}
defer file.Close()
for _, stream := range block.NewStreams {
n, err := file.Write(stream)
if err != nil || n < 1 {
return err
}
varName := variable.ProtoToString(stream.Variable)
existingstream, found := block.LogStreams[varName]
if found {
existingstream.Value = append(existingstream.Value, stream.Value...)
} else {
block.LogStreams[varName] = stream
}
}
block.NewStreams = make([]*oproto.ValueStream, 0)
block.Block.LoggedStreams += block.Block.UnloggedStreams
block.Block.LoggedValues += block.Block.UnloggedValues
block.Block.UnloggedStreams = uint32(0)
block.Block.UnloggedValues = uint32(0)
block.UpdateSize()
return nil
}
// Write writes a map of ValueStreams to a single block file on disk.
// The values inside each ValueStream will be sorted and run-length-encoded before writing.
func (block *Block) Write(ctx context.Context, streams map[string]*oproto.ValueStream) error {
// Build the header with a 0-index for each variable
block.Block.Header.Index = []*oproto.BlockHeaderIndex{}
block.Block.Header.EndKey = ""
block.Block.Header.StartTimestamp = 0
block.Block.Header.EndTimestamp = 0
streams = block.RunLengthEncodeStreams(ctx, streams)
for v, stream := range streams {
if v > block.Block.Header.EndKey {
block.Block.Header.EndKey = v
}
// Add this stream to the index
block.Block.Header.Index = append(block.Block.Header.Index, &oproto.BlockHeaderIndex{
Variable: stream.Variable,
Offset: uint64(1), // This must be set non-zero so that the protobuf marshals it to non-empty
MinTimestamp: stream.Value[0].Timestamp,
MaxTimestamp: stream.Value[len(stream.Value)-1].Timestamp,
NumValues: uint32(len(stream.Value)),
})
if block.Block.Header.StartTimestamp == 0 || stream.Value[0].Timestamp < block.Block.Header.StartTimestamp {
block.Block.Header.StartTimestamp = stream.Value[0].Timestamp
}
if stream.Value[len(stream.Value)-1].Timestamp > block.Block.Header.EndTimestamp {
block.Block.Header.EndTimestamp = stream.Value[len(stream.Value)-1].Timestamp
}
}
// Start writing to the new block file
newfilename := fmt.Sprintf("%s.new.%d", block.Filename(), os.Getpid())
newfile, err := protofile.Write(newfilename)
if err != nil {
newfile.Close()
return fmt.Errorf("Can't write to %s: %s\n", newfilename, err)
}
newfile.Write(block.Block.Header)
blockEnd := newfile.Tell()
// Write all the ValueStreams
indexPos := make(map[string]uint64)
var outValues uint32
for _, stream := range streams {
indexPos[variable.ProtoToString(stream.Variable)] = uint64(newfile.Tell())
newfile.Write(stream)
outValues += uint32(len(stream.Value))
}
// Update the offsets in the header, now that all the data has been written
for _, index := range block.Block.Header.Index {
index.Offset = indexPos[variable.ProtoToString(index.Variable)]
}
newfile.WriteAt(0, block.Block.Header)
if blockEnd < newfile.Tell() {
// Sanity check, just in case goprotobuf breaks something again
newfile.Close()
os.Remove(newfilename)
log.Fatalf("Error writing block file %s, header overwrote data", newfilename)
}
newfile.Sync()
newfile.Close()
block.UpdateIndexedCount()
openinstrument.Logf(ctx, "Wrote %d streams / %d values to %s", len(streams), outValues, newfilename)
openinstrument.Logf(ctx, "Block log contains %d stream", len(block.Block.Header.Index))
// Rename the temporary file into place
if err := os.Rename(newfilename, block.Filename()); err != nil {
return fmt.Errorf("Error renaming: %s", err)
}
return nil
}