func (ds *Datastore) readBlockHeader(ctx context.Context, filename string) { block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path) file, err := protofile.Read(block.Filename()) if err != nil { openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.Filename(), err) return } defer file.Close() if n, err := file.Read(block.Block.Header); n < 1 || err != nil { openinstrument.Logf(ctx, "Block file %s has a corrupted header: %s\n", block.Filename(), err) return } if block.Block.Header.Version != 2 { openinstrument.Logf(ctx, "Block file %s has incorrect version identifier '%v'\n", block.Filename(), block.Block.Header.Version) return } block.Block.EndKey = block.Block.Header.EndKey if block.EndKey() == "" { openinstrument.Logf(ctx, "Block %s does not have an end key, ignoring", block.Filename()) return } // Update cached number of streams and values block.UpdateIndexedCount() ds.insertBlock(ctx, block) openinstrument.Logf(ctx, "Read block %s containing %d streams\n", block.ID(), len(block.Block.Header.Index)) }
func (block *Block) GetLoggedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) { file, err := protofile.Read(block.logFilename()) if err != nil { return nil, fmt.Errorf("Can't read block log file %s: %s\n", block.logFilename(), err) } return file.ValueStreamReader(ctx, 5000), nil }
func (block *Block) getIndexedStream(ctx context.Context, index *oproto.BlockHeaderIndex) *oproto.ValueStream { file, err := protofile.Read(block.Filename()) if err != nil { if !os.IsNotExist(err) { openinstrument.Logf(ctx, "Can't read block file %s: %s\n", block, err) } return nil } defer file.Close() stream := &oproto.ValueStream{} if n, err := file.ReadAt(int64(index.Offset), stream); n < 1 || err != nil { openinstrument.Logf(ctx, "Couldn't read ValueStream at %s:%d: %s", block, index.Offset, err) return nil } return stream }
func (ds *Datastore) readBlockLog(ctx context.Context, filename string) { block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path) file, err := protofile.Read(block.logFilename()) if err != nil { openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.logFilename(), err) } defer file.Close() // Read all the streams from the log file reader := file.ValueStreamReader(ctx, 100) for stream := range reader { varName := variable.ProtoToString(stream.Variable) if varName > block.EndKey() { block.Block.EndKey = varName } locker := block.LogWriteLocker() locker.Lock() existingstream, found := block.LogStreams[varName] if found { existingstream.Value = append(existingstream.Value, stream.Value...) } else { block.LogStreams[varName] = stream } locker.Unlock() } if func() *Block { for _, existingblock := range ds.Blocks() { if existingblock.ID() == block.ID() { locker := existingblock.LogWriteLocker() locker.Lock() existingblock.LogStreams = block.LogStreams locker.Unlock() // Update cached number of streams and values existingblock.UpdateLoggedCount() return existingblock } } return nil }() == nil { // There is no existing block file for this log. block.UpdateLoggedCount() ds.insertBlock(ctx, block) } }
func (block *Block) GetIndexedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) { block.UpdateSize() file, err := protofile.Read(block.Filename()) if err != nil { return nil, fmt.Errorf("Can't read block file %s: %s\n", block.Filename(), err) } n, err := file.Read(block.Block.Header) if n < 1 || err != nil { file.Close() return nil, fmt.Errorf("Block %s has a corrupted header: %s\n", block.Filename(), err) } switch block.Block.Header.Version { case 2: return file.ValueStreamReader(ctx, 5000), nil default: return nil, fmt.Errorf("Block %s has unknown version '%v'\n", block.Filename(), block.Block.Header.Version) } }