Esempio n. 1
0
func (ds *Datastore) readBlockHeader(ctx context.Context, filename string) {
	block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path)

	file, err := protofile.Read(block.Filename())
	if err != nil {
		openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.Filename(), err)
		return
	}
	defer file.Close()

	if n, err := file.Read(block.Block.Header); n < 1 || err != nil {
		openinstrument.Logf(ctx, "Block file %s has a corrupted header: %s\n", block.Filename(), err)
		return
	}

	if block.Block.Header.Version != 2 {
		openinstrument.Logf(ctx, "Block file %s has incorrect version identifier '%v'\n", block.Filename(), block.Block.Header.Version)
		return
	}

	block.Block.EndKey = block.Block.Header.EndKey
	if block.EndKey() == "" {
		openinstrument.Logf(ctx, "Block %s does not have an end key, ignoring", block.Filename())
		return
	}

	// Update cached number of streams and values
	block.UpdateIndexedCount()

	ds.insertBlock(ctx, block)
	openinstrument.Logf(ctx, "Read block %s containing %d streams\n", block.ID(), len(block.Block.Header.Index))
}
Esempio n. 2
0
func (block *Block) GetLoggedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) {
	file, err := protofile.Read(block.logFilename())
	if err != nil {
		return nil, fmt.Errorf("Can't read block log file %s: %s\n", block.logFilename(), err)
	}
	return file.ValueStreamReader(ctx, 5000), nil
}
Esempio n. 3
0
func (block *Block) getIndexedStream(ctx context.Context, index *oproto.BlockHeaderIndex) *oproto.ValueStream {
	file, err := protofile.Read(block.Filename())
	if err != nil {
		if !os.IsNotExist(err) {
			openinstrument.Logf(ctx, "Can't read block file %s: %s\n", block, err)
		}
		return nil
	}
	defer file.Close()
	stream := &oproto.ValueStream{}
	if n, err := file.ReadAt(int64(index.Offset), stream); n < 1 || err != nil {
		openinstrument.Logf(ctx, "Couldn't read ValueStream at %s:%d: %s", block, index.Offset, err)
		return nil
	}
	return stream
}
Esempio n. 4
0
func (ds *Datastore) readBlockLog(ctx context.Context, filename string) {
	block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path)

	file, err := protofile.Read(block.logFilename())
	if err != nil {
		openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.logFilename(), err)
	}
	defer file.Close()

	// Read all the streams from the log file
	reader := file.ValueStreamReader(ctx, 100)
	for stream := range reader {
		varName := variable.ProtoToString(stream.Variable)
		if varName > block.EndKey() {
			block.Block.EndKey = varName
		}
		locker := block.LogWriteLocker()
		locker.Lock()
		existingstream, found := block.LogStreams[varName]
		if found {
			existingstream.Value = append(existingstream.Value, stream.Value...)
		} else {
			block.LogStreams[varName] = stream
		}
		locker.Unlock()
	}

	if func() *Block {
		for _, existingblock := range ds.Blocks() {
			if existingblock.ID() == block.ID() {
				locker := existingblock.LogWriteLocker()
				locker.Lock()
				existingblock.LogStreams = block.LogStreams
				locker.Unlock()
				// Update cached number of streams and values
				existingblock.UpdateLoggedCount()
				return existingblock
			}
		}
		return nil
	}() == nil {
		// There is no existing block file for this log.
		block.UpdateLoggedCount()
		ds.insertBlock(ctx, block)
	}
}
Esempio n. 5
0
func (block *Block) GetIndexedStreams(ctx context.Context) (<-chan *oproto.ValueStream, error) {
	block.UpdateSize()
	file, err := protofile.Read(block.Filename())
	if err != nil {
		return nil, fmt.Errorf("Can't read block file %s: %s\n", block.Filename(), err)
	}

	n, err := file.Read(block.Block.Header)
	if n < 1 || err != nil {
		file.Close()
		return nil, fmt.Errorf("Block %s has a corrupted header: %s\n", block.Filename(), err)
	}
	switch block.Block.Header.Version {
	case 2:
		return file.ValueStreamReader(ctx, 5000), nil
	default:
		return nil, fmt.Errorf("Block %s has unknown version '%v'\n", block.Filename(), block.Block.Header.Version)
	}
}