func (block *Block) Flush() error { block.newStreamsLock.Lock() defer block.newStreamsLock.Unlock() if len(block.NewStreams) == 0 { return nil } block.logLock.Lock() defer block.logLock.Unlock() // There are streams that need to be flushed to disk file, err := protofile.Write(block.logFilename()) if err != nil { return err } defer file.Close() for _, stream := range block.NewStreams { n, err := file.Write(stream) if err != nil || n < 1 { return err } varName := variable.ProtoToString(stream.Variable) existingstream, found := block.LogStreams[varName] if found { existingstream.Value = append(existingstream.Value, stream.Value...) } else { block.LogStreams[varName] = stream } } block.NewStreams = make([]*oproto.ValueStream, 0) block.Block.LoggedStreams += block.Block.UnloggedStreams block.Block.LoggedValues += block.Block.UnloggedValues block.Block.UnloggedStreams = uint32(0) block.Block.UnloggedValues = uint32(0) block.UpdateSize() return nil }
// Write writes a map of ValueStreams to a single block file on disk. // The values inside each ValueStream will be sorted and run-length-encoded before writing. func (block *Block) Write(ctx context.Context, streams map[string]*oproto.ValueStream) error { // Build the header with a 0-index for each variable block.Block.Header.Index = []*oproto.BlockHeaderIndex{} block.Block.Header.EndKey = "" block.Block.Header.StartTimestamp = 0 block.Block.Header.EndTimestamp = 0 streams = block.RunLengthEncodeStreams(ctx, streams) for v, stream := range streams { if v > block.Block.Header.EndKey { block.Block.Header.EndKey = v } // Add this stream to the index block.Block.Header.Index = append(block.Block.Header.Index, &oproto.BlockHeaderIndex{ Variable: stream.Variable, Offset: uint64(1), // This must be set non-zero so that the protobuf marshals it to non-empty MinTimestamp: stream.Value[0].Timestamp, MaxTimestamp: stream.Value[len(stream.Value)-1].Timestamp, NumValues: uint32(len(stream.Value)), }) if block.Block.Header.StartTimestamp == 0 || stream.Value[0].Timestamp < block.Block.Header.StartTimestamp { block.Block.Header.StartTimestamp = stream.Value[0].Timestamp } if stream.Value[len(stream.Value)-1].Timestamp > block.Block.Header.EndTimestamp { block.Block.Header.EndTimestamp = stream.Value[len(stream.Value)-1].Timestamp } } // Start writing to the new block file newfilename := fmt.Sprintf("%s.new.%d", block.Filename(), os.Getpid()) newfile, err := protofile.Write(newfilename) if err != nil { newfile.Close() return fmt.Errorf("Can't write to %s: %s\n", newfilename, err) } newfile.Write(block.Block.Header) blockEnd := newfile.Tell() // Write all the ValueStreams indexPos := make(map[string]uint64) var outValues uint32 for _, stream := range streams { indexPos[variable.ProtoToString(stream.Variable)] = uint64(newfile.Tell()) newfile.Write(stream) outValues += uint32(len(stream.Value)) } // Update the offsets in the header, now that all the data has been written for _, index := range block.Block.Header.Index { index.Offset = indexPos[variable.ProtoToString(index.Variable)] } newfile.WriteAt(0, block.Block.Header) if blockEnd < newfile.Tell() { // Sanity check, just in case goprotobuf breaks something again newfile.Close() os.Remove(newfilename) log.Fatalf("Error writing block file %s, header overwrote data", newfilename) } newfile.Sync() newfile.Close() block.UpdateIndexedCount() openinstrument.Logf(ctx, "Wrote %d streams / %d values to %s", len(streams), outValues, newfilename) openinstrument.Logf(ctx, "Block log contains %d stream", len(block.Block.Header.Index)) // Rename the temporary file into place if err := os.Rename(newfilename, block.Filename()); err != nil { return fmt.Errorf("Error renaming: %s", err) } return nil }