func (ds *Datastore) readBlockHeader(ctx context.Context, filename string) {
block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path)
file, err := protofile.Read(block.Filename())
if err != nil {
openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.Filename(), err)
return
}
defer file.Close()
if n, err := file.Read(block.Block.Header); n < 1 || err != nil {
openinstrument.Logf(ctx, "Block file %s has a corrupted header: %s\n", block.Filename(), err)
return
}
if block.Block.Header.Version != 2 {
openinstrument.Logf(ctx, "Block file %s has incorrect version identifier '%v'\n", block.Filename(), block.Block.Header.Version)
return
}
block.Block.EndKey = block.Block.Header.EndKey
if block.EndKey() == "" {
openinstrument.Logf(ctx, "Block %s does not have an end key, ignoring", block.Filename())
return
}
// Update cached number of streams and values
block.UpdateIndexedCount()
ds.insertBlock(ctx, block)
openinstrument.Logf(ctx, "Read block %s containing %d streams\n", block.ID(), len(block.Block.Header.Index))
}
func (block *Block) GetLoggedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) {
file, err := protofile.Read(block.logFilename())
if err != nil {
return nil, fmt.Errorf("Can't read block log file %s: %s\n", block.logFilename(), err)
}
return file.ValueStreamReader(ctx, 5000), nil
}
func (block *Block) getIndexedStream(ctx context.Context, index *oproto.BlockHeaderIndex) *oproto.ValueStream {
file, err := protofile.Read(block.Filename())
if err != nil {
if !os.IsNotExist(err) {
openinstrument.Logf(ctx, "Can't read block file %s: %s\n", block, err)
}
return nil
}
defer file.Close()
stream := &oproto.ValueStream{}
if n, err := file.ReadAt(int64(index.Offset), stream); n < 1 || err != nil {
openinstrument.Logf(ctx, "Couldn't read ValueStream at %s:%d: %s", block, index.Offset, err)
return nil
}
return stream
}
func (ds *Datastore) readBlockLog(ctx context.Context, filename string) {
block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path)
file, err := protofile.Read(block.logFilename())
if err != nil {
openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.logFilename(), err)
}
defer file.Close()
// Read all the streams from the log file
reader := file.ValueStreamReader(ctx, 100)
for stream := range reader {
varName := variable.ProtoToString(stream.Variable)
if varName > block.EndKey() {
block.Block.EndKey = varName
}
locker := block.LogWriteLocker()
locker.Lock()
existingstream, found := block.LogStreams[varName]
if found {
existingstream.Value = append(existingstream.Value, stream.Value...)
} else {
block.LogStreams[varName] = stream
}
locker.Unlock()
}
if func() *Block {
for _, existingblock := range ds.Blocks() {
if existingblock.ID() == block.ID() {
locker := existingblock.LogWriteLocker()
locker.Lock()
existingblock.LogStreams = block.LogStreams
locker.Unlock()
// Update cached number of streams and values
existingblock.UpdateLoggedCount()
return existingblock
}
}
return nil
}() == nil {
// There is no existing block file for this log.
block.UpdateLoggedCount()
ds.insertBlock(ctx, block)
}
}
func (block *Block) GetIndexedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) {
block.UpdateSize()
file, err := protofile.Read(block.Filename())
if err != nil {
return nil, fmt.Errorf("Can't read block file %s: %s\n", block.Filename(), err)
}
n, err := file.Read(block.Block.Header)
if n < 1 || err != nil {
file.Close()
return nil, fmt.Errorf("Block %s has a corrupted header: %s\n", block.Filename(), err)
}
switch block.Block.Header.Version {
case 2:
return file.ValueStreamReader(ctx, 5000), nil
default:
return nil, fmt.Errorf("Block %s has unknown version '%v'\n", block.Filename(), block.Block.Header.Version)
}
}