Esempio n. 1
0
func (block *Block) Flush() error {
	block.newStreamsLock.Lock()
	defer block.newStreamsLock.Unlock()

	if len(block.NewStreams) == 0 {
		return nil
	}

	block.logLock.Lock()
	defer block.logLock.Unlock()

	// There are streams that need to be flushed to disk
	file, err := protofile.Write(block.logFilename())
	if err != nil {
		return err
	}
	defer file.Close()
	for _, stream := range block.NewStreams {
		n, err := file.Write(stream)
		if err != nil || n < 1 {
			return err
		}
		varName := variable.ProtoToString(stream.Variable)
		existingstream, found := block.LogStreams[varName]
		if found {
			existingstream.Value = append(existingstream.Value, stream.Value...)
		} else {
			block.LogStreams[varName] = stream
		}
	}
	block.NewStreams = make([]*oproto.ValueStream, 0)
	block.Block.LoggedStreams += block.Block.UnloggedStreams
	block.Block.LoggedValues += block.Block.UnloggedValues
	block.Block.UnloggedStreams = uint32(0)
	block.Block.UnloggedValues = uint32(0)
	block.UpdateSize()

	return nil
}
Esempio n. 2
0
// Write writes a map of ValueStreams to a single block file on disk.
// The values inside each ValueStream will be sorted and run-length-encoded before writing.
func (block *Block) Write(ctx context.Context, streams map[string]*oproto.ValueStream) error {
	// Build the header with a 0-index for each variable
	block.Block.Header.Index = []*oproto.BlockHeaderIndex{}
	block.Block.Header.EndKey = ""
	block.Block.Header.StartTimestamp = 0
	block.Block.Header.EndTimestamp = 0
	streams = block.RunLengthEncodeStreams(ctx, streams)
	for v, stream := range streams {
		if v > block.Block.Header.EndKey {
			block.Block.Header.EndKey = v
		}
		// Add this stream to the index
		block.Block.Header.Index = append(block.Block.Header.Index, &oproto.BlockHeaderIndex{
			Variable:     stream.Variable,
			Offset:       uint64(1), // This must be set non-zero so that the protobuf marshals it to non-empty
			MinTimestamp: stream.Value[0].Timestamp,
			MaxTimestamp: stream.Value[len(stream.Value)-1].Timestamp,
			NumValues:    uint32(len(stream.Value)),
		})

		if block.Block.Header.StartTimestamp == 0 || stream.Value[0].Timestamp < block.Block.Header.StartTimestamp {
			block.Block.Header.StartTimestamp = stream.Value[0].Timestamp
		}
		if stream.Value[len(stream.Value)-1].Timestamp > block.Block.Header.EndTimestamp {
			block.Block.Header.EndTimestamp = stream.Value[len(stream.Value)-1].Timestamp
		}
	}

	// Start writing to the new block file
	newfilename := fmt.Sprintf("%s.new.%d", block.Filename(), os.Getpid())
	newfile, err := protofile.Write(newfilename)
	if err != nil {
		newfile.Close()
		return fmt.Errorf("Can't write to %s: %s\n", newfilename, err)
	}
	newfile.Write(block.Block.Header)
	blockEnd := newfile.Tell()

	// Write all the ValueStreams
	indexPos := make(map[string]uint64)
	var outValues uint32
	for _, stream := range streams {
		indexPos[variable.ProtoToString(stream.Variable)] = uint64(newfile.Tell())
		newfile.Write(stream)
		outValues += uint32(len(stream.Value))
	}

	// Update the offsets in the header, now that all the data has been written
	for _, index := range block.Block.Header.Index {
		index.Offset = indexPos[variable.ProtoToString(index.Variable)]
	}

	newfile.WriteAt(0, block.Block.Header)
	if blockEnd < newfile.Tell() {
		// Sanity check, just in case goprotobuf breaks something again
		newfile.Close()
		os.Remove(newfilename)
		log.Fatalf("Error writing block file %s, header overwrote data", newfilename)
	}

	newfile.Sync()
	newfile.Close()

	block.UpdateIndexedCount()

	openinstrument.Logf(ctx, "Wrote %d streams / %d values to %s", len(streams), outValues, newfilename)
	openinstrument.Logf(ctx, "Block log contains %d stream", len(block.Block.Header.Index))

	// Rename the temporary file into place
	if err := os.Rename(newfilename, block.Filename()); err != nil {
		return fmt.Errorf("Error renaming: %s", err)
	}

	return nil
}