func (s *MySuite) TestAverage(c *C) {
ctx, cancel := context.WithCancel(context.Background())
e := NewExporter(ctx, s.Client, 10*time.Second)
a := NewAverage(e, variable.NewFromString("/test/average"))
a.Update(95, 1)
a.Update(100, 1)
a.Update(105, 1)
// Force export
streams := e.GetStreams()
c.Assert(len(streams), Equals, 2)
for _, stream := range streams {
switch variable.ProtoToString(stream.Variable) {
case "/test/average-total-count":
c.Check(stream.Value[0].DoubleValue, Equals, 300.0)
case "/test/average-overall-sum":
c.Check(stream.Value[0].DoubleValue, Equals, 3.0)
default:
fmt.Printf("Invalid variable %s", variable.ProtoToString(stream.Variable))
c.Fail()
}
}
cancel()
<-ctx.Done()
}
func (s *MySuite) TestMean(c *C) {
q, err := Parse("mean by (xyz) (/test{host=a}, /test{host=b})")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Variable[0]), Equals, "/test{host=a}")
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Variable[1]), Equals, "/test{host=b}")
c.Check(query.Aggregation[0].Type, Equals, oproto.StreamAggregation_MEAN)
c.Check(query.Aggregation[0].Label[0], Equals, "xyz")
ch, err := q.Run(context.Background(), s.store)
c.Assert(err, IsNil)
output := []*oproto.ValueStream{}
for stream := range ch {
output = append(output, stream)
}
c.Check(output[0].Value[0].DoubleValue, Equals, float64((20*1+40*1)/2))
c.Check(output[0].Value[1].DoubleValue, Equals, float64((20*2+40*2)/2))
c.Check(output[0].Value[2].DoubleValue, Equals, float64((20*3+40*3)/2))
c.Check(output[0].Value[3].DoubleValue, Equals, float64((20*4+40*4)/2))
c.Check(output[0].Value[4].DoubleValue, Equals, float64((20*5+40*5)/2))
c.Check(output[0].Value[5].DoubleValue, Equals, float64((20*6+40*6)/2))
c.Check(output[0].Value[6].DoubleValue, Equals, float64((20*7+40*7)/2))
c.Check(output[0].Value[7].DoubleValue, Equals, float64((20*8+40*8)/2))
c.Check(output[0].Value[8].DoubleValue, Equals, float64((20*9+40*9)/2))
c.Check(output[0].Value[9].DoubleValue, Equals, float64((20*10+40*10)/2))
c.Check(output[0].Value[10].DoubleValue, Equals, float64((20*11+40*11)/2))
}
func (s *MySuite) TestRatio(c *C) {
ctx, cancel := context.WithCancel(context.Background())
e := NewExporter(ctx, s.Client, 10*time.Second)
r := NewRatio(e, variable.NewFromString("/test/ratio"))
for i := 0; i < 10; i++ {
r.Success()
}
for i := 0; i < 5; i++ {
r.Failure()
}
// Force export
streams := e.GetStreams()
c.Assert(len(streams), Equals, 3)
for _, stream := range streams {
switch variable.ProtoToString(stream.Variable) {
case "/test/ratio-success":
c.Check(stream.Value[0].DoubleValue, Equals, 10.0)
case "/test/ratio-failure":
c.Check(stream.Value[0].DoubleValue, Equals, 5.0)
case "/test/ratio-total":
c.Check(stream.Value[0].DoubleValue, Equals, 15.0)
default:
fmt.Printf("Invalid variable %s", variable.ProtoToString(stream.Variable))
c.Fail()
}
}
cancel()
<-ctx.Done()
}
func (s *MySuite) TestTimer(c *C) {
ctx, cancel := context.WithCancel(context.Background())
e := NewExporter(ctx, s.Client, 10*time.Second)
t := NewTimer(e, variable.NewFromString("/test/timer"))
t.Start()
time.Sleep(10 * time.Millisecond)
t.Stop()
t.Start()
time.Sleep(10 * time.Millisecond)
t.Stop()
// Force export
streams := e.GetStreams()
c.Assert(len(streams), Equals, 2)
for _, stream := range streams {
switch variable.ProtoToString(stream.Variable) {
case "/test/timer-total-count":
c.Check((stream.Value[0].DoubleValue >= 20.0 && stream.Value[0].DoubleValue <= 25.0), Equals, true)
case "/test/timer-overall-sum":
c.Check(stream.Value[0].DoubleValue, Equals, 2.0)
default:
fmt.Printf("Invalid variable %s", variable.ProtoToString(stream.Variable))
c.Fail()
}
}
cancel()
<-ctx.Done()
}
func (s *MySuite) TestAggregationOfMutations(c *C) {
q, err := Parse("mean by (host) (rate(/test{host=a}[1200:1500], /test{host=b}[1200:1500]))")
c.Assert(err, IsNil)
query := q.query
c.Check(query.Aggregation[0].Type, Equals, oproto.StreamAggregation_MEAN)
c.Check(query.Aggregation[0].Query[0].Mutation[0].Type, Equals, oproto.StreamMutation_RATE)
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0]), Equals, "/test{host=a}[1200:1500]")
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1]), Equals, "/test{host=b}[1200:1500]")
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0].MinTimestamp, Equals, int64(1200))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0].MaxTimestamp, Equals, int64(1500))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1].MinTimestamp, Equals, int64(1200))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1].MaxTimestamp, Equals, int64(1500))
}
func (s *MySuite) TestAggregationOfPercentile(c *C) {
q, err := Parse("percentile(90) by (host) (rate(/test{host=a}[1200:1500], /test{host=b}[1200:1500]))")
c.Assert(err, IsNil)
query := q.query
c.Check(query.Aggregation[0].Type, Equals, oproto.StreamAggregation_PERCENTILE)
c.Check(query.Aggregation[0].Param, Equals, 90.0)
c.Check(query.Aggregation[0].Label[0], Equals, "host")
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0]), Equals, "/test{host=a}[1200:1500]")
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1]), Equals, "/test{host=b}[1200:1500]")
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0].MinTimestamp, Equals, int64(1200))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[0].MaxTimestamp, Equals, int64(1500))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1].MinTimestamp, Equals, int64(1200))
c.Check(query.Aggregation[0].Query[0].Mutation[0].Query.Variable[1].MaxTimestamp, Equals, int64(1500))
}
// AddStream adds a new stream to the unlogged streams list.
// The stream is not flushed to disk until block.Flush() is called (which happens regularly).
func (block *Block) AddStream(stream *oproto.ValueStream) {
block.newStreamsLock.Lock()
defer block.newStreamsLock.Unlock()
v := variable.ProtoToString(stream.Variable)
for _, existingstream := range block.NewStreams {
if variable.ProtoToString(existingstream.Variable) == v {
existingstream.Value = append(existingstream.Value, stream.Value...)
block.Block.UnloggedValues += uint32(len(stream.Value))
return
}
}
block.NewStreams = append(block.NewStreams, stream)
block.Block.UnloggedValues += uint32(len(stream.Value))
block.Block.UnloggedStreams++
}
func (s *MySuite) TestGetStreamForVariable(c *C) {
block := NewBlock(context.Background(), "/test/foo", "", s.dataDir)
for v := 0; v < 10; v++ {
varName := fmt.Sprintf("/test/bar%d", v)
block.LogStreams[varName] = &oproto.ValueStream{
Variable: &oproto.StreamVariable{Name: varName},
Value: []*oproto.Value{},
}
for i := 0; i < 100; i++ {
block.LogStreams[varName].Value = append(block.LogStreams[varName].Value,
&oproto.Value{DoubleValue: float64(i)})
}
}
c.Assert(block.Compact(context.Background()), IsNil)
found := false
for _, index := range block.Block.Header.Index {
cv := variable.NewFromProto(index.Variable)
if cv.String() != "/test/bar7" {
continue
}
stream := block.getIndexedStream(context.Background(), index)
c.Assert(variable.ProtoToString(stream.Variable), Equals, "/test/bar7")
found = true
break
}
c.Assert(found, Equals, true)
}
func (block *Block) RunLengthEncodeStreams(ctx context.Context, streams map[string]*oproto.ValueStream) map[string]*oproto.ValueStream {
// Run-length encode all streams in parallel
var sl sync.Mutex
var outputValues int
wg := &sync.WaitGroup{}
newStreams := make(map[string]*oproto.ValueStream, 0)
for _, stream := range streams {
wg.Add(1)
go func(stream *oproto.ValueStream) {
defer wg.Done()
// Sort values by timestamp
value.By(func(a, b *oproto.Value) bool { return a.Timestamp < b.Timestamp }).Sort(stream.Value)
// Run-length encode values
stream = rle.Encode(stream)
sl.Lock()
newStreams[variable.ProtoToString(stream.Variable)] = stream
outputValues += len(stream.Value)
sl.Unlock()
}(stream)
}
wg.Wait()
openinstrument.Logf(ctx, "Run-length encoded %d streams to %d", len(newStreams), outputValues)
return newStreams
}
func (s *MySuite) TestMutation(c *C) {
q, err := Parse("rate(/test{host=a})")
c.Assert(err, IsNil)
query := q.query
c.Check(query.Mutation[0].Type, Equals, oproto.StreamMutation_RATE)
c.Check(variable.ProtoToString(query.Mutation[0].Query.Variable[0]), Equals, "/test{host=a}")
ch, err := q.Run(context.Background(), s.store)
c.Assert(err, IsNil)
numStreams := 0
for stream := range ch {
c.Check(variable.ProtoToString(stream.Variable), Equals, "/test{host=a}")
c.Check(len(stream.Value), Equals, 10)
numStreams++
}
c.Check(numStreams, Equals, 1)
}
func (s *MySuite) TestVariableNoLabels(c *C) {
q, err := Parse("/test{}")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Variable[0]), Equals, "/test")
c.Check(query.Variable[0].MinTimestamp, Equals, int64(0))
c.Check(query.Variable[0].MaxTimestamp, Equals, int64(0))
ch, err := q.Run(context.Background(), s.store)
c.Assert(err, IsNil)
numStreams := 0
for stream := range ch {
c.Check(variable.ProtoToString(stream.Variable), Equals, "/test")
c.Check(len(stream.Value), Equals, 11)
numStreams++
}
c.Check(numStreams, Equals, 1)
}
func (s *MySuite) TestPercentile(c *C) {
q, err := Parse("percentile(20) by (host) (/test{host=a})")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Aggregation[0].Query[0].Variable[0]), Equals, "/test{host=a}")
c.Check(query.Aggregation[0].Type, Equals, oproto.StreamAggregation_PERCENTILE)
c.Check(query.Aggregation[0].Param, Equals, 20.0)
c.Check(query.Aggregation[0].Label[0], Equals, "host")
}
func (block *Block) AddStreams(c <-chan *oproto.ValueStream) {
block.newStreamsLock.Lock()
defer block.newStreamsLock.Unlock()
CHAN:
for stream := range c {
v := variable.ProtoToString(stream.Variable)
for _, existingstream := range block.NewStreams {
if variable.ProtoToString(existingstream.Variable) == v {
existingstream.Value = append(existingstream.Value, stream.Value...)
block.Block.UnloggedValues += uint32(len(stream.Value))
continue CHAN
}
}
block.NewStreams = append(block.NewStreams, stream)
block.Block.UnloggedValues += uint32(len(stream.Value))
block.Block.UnloggedStreams++
}
}
func (s *MySuite) TestFloat(c *C) {
ctx, cancel := context.WithCancel(context.Background())
e := NewExporter(ctx, s.Client, 10*time.Second)
f := NewFloat(e, variable.NewFromString("/test/float"))
f.Set(100.0)
// Force export
streams := e.GetStreams()
c.Assert(len(streams), Equals, 1)
c.Check(variable.ProtoToString(streams[0].Variable), Equals, "/test/float")
c.Check(streams[0].Value[0].DoubleValue, Equals, 100.0)
cancel()
<-ctx.Done()
}
// Writer builds a channel that can accept ValueStreams for writing to the datastore.
// Any ValueStreams written to this channel will eventually be flushed to disk,
// but they will be immediately available for use.
// The writes to disk are not guaranteed until Flush is called.
func (ds *Datastore) Writer(ctx context.Context) chan<- *oproto.ValueStream {
in := make(chan *oproto.ValueStream, 10)
go func() {
for stream := range in {
// Write this stream
varName := variable.ProtoToString(stream.Variable)
if block := ds.findBlock(ctx, varName); block != nil {
//openinstrument.Logf(ctx, "Writing stream for variable %s to block %s", varName, block.ID())
block.AddStream(stream)
} else {
openinstrument.Logf(ctx, "Unable to find block to write variable %s", varName)
}
}
}()
return in
}
func (s *server) LookupBlock(ctx context.Context, request *oproto.LookupBlockRequest) (*oproto.LookupBlockResponse, error) {
if request.BlockId != "" {
block, err := s.ds.GetBlock(request.BlockId, "")
if err != nil {
return nil, err
}
return &oproto.LookupBlockResponse{Block: block.ToProto()}, nil
}
v := variable.ProtoToString(request.Variable)
for _, block := range s.ds.Blocks() {
if block.EndKey() >= v {
return &oproto.LookupBlockResponse{Block: block.ToProto()}, nil
}
}
return &oproto.LookupBlockResponse{}, nil
}
func InspectVariable(ctx context.Context, ds *datastore.Datastore, w http.ResponseWriter, req *http.Request) {
t, err := template.ParseFiles(fmt.Sprintf("%s/inspect_variable.html", *templatePath))
if err != nil {
openinstrument.Logf(ctx, "Couldn't find template file: %s", err)
return
}
type varInfo struct {
Name string
FirstTimestamp time.Time
LastTimestamp time.Time
}
p := struct {
Title string
Query string
Variables []varInfo
}{
Title: "Inspect Variable",
Query: req.FormValue("q"),
Variables: make([]varInfo, 0),
}
if p.Query == "" {
w.WriteHeader(404)
fmt.Fprintf(w, "Specify q=")
return
}
v := variable.NewFromString(p.Query)
c := ds.Reader(ctx, v)
for stream := range c {
lt := stream.Value[len(stream.Value)-1].EndTimestamp
if lt == 0 {
lt = stream.Value[len(stream.Value)-1].Timestamp
}
p.Variables = append(p.Variables, varInfo{
Name: variable.ProtoToString(stream.Variable),
FirstTimestamp: time.Unix(int64(stream.Value[0].Timestamp/1000), 0),
LastTimestamp: time.Unix(int64(lt/1000), 0),
})
}
err = t.Execute(w, p)
if err != nil {
log.Println(err)
}
}
func (ds *Datastore) readBlockLog(ctx context.Context, filename string) {
block := NewBlock(ctx, "", BlockIDFromFilename(filename), ds.Path)
file, err := protofile.Read(block.logFilename())
if err != nil {
openinstrument.Logf(ctx, "Error opening proto log file %s: %s", block.logFilename(), err)
}
defer file.Close()
// Read all the streams from the log file
reader := file.ValueStreamReader(ctx, 100)
for stream := range reader {
varName := variable.ProtoToString(stream.Variable)
if varName > block.EndKey() {
block.Block.EndKey = varName
}
locker := block.LogWriteLocker()
locker.Lock()
existingstream, found := block.LogStreams[varName]
if found {
existingstream.Value = append(existingstream.Value, stream.Value...)
} else {
block.LogStreams[varName] = stream
}
locker.Unlock()
}
if func() *Block {
for _, existingblock := range ds.Blocks() {
if existingblock.ID() == block.ID() {
locker := existingblock.LogWriteLocker()
locker.Lock()
existingblock.LogStreams = block.LogStreams
locker.Unlock()
// Update cached number of streams and values
existingblock.UpdateLoggedCount()
return existingblock
}
}
return nil
}() == nil {
// There is no existing block file for this log.
block.UpdateLoggedCount()
ds.insertBlock(ctx, block)
}
}
func (block *Block) Flush() error {
block.newStreamsLock.Lock()
defer block.newStreamsLock.Unlock()
if len(block.NewStreams) == 0 {
return nil
}
block.logLock.Lock()
defer block.logLock.Unlock()
// There are streams that need to be flushed to disk
file, err := protofile.Write(block.logFilename())
if err != nil {
return err
}
defer file.Close()
for _, stream := range block.NewStreams {
n, err := file.Write(stream)
if err != nil || n < 1 {
return err
}
varName := variable.ProtoToString(stream.Variable)
existingstream, found := block.LogStreams[varName]
if found {
existingstream.Value = append(existingstream.Value, stream.Value...)
} else {
block.LogStreams[varName] = stream
}
}
block.NewStreams = make([]*oproto.ValueStream, 0)
block.Block.LoggedStreams += block.Block.UnloggedStreams
block.Block.LoggedValues += block.Block.UnloggedValues
block.Block.UnloggedStreams = uint32(0)
block.Block.UnloggedValues = uint32(0)
block.UpdateSize()
return nil
}
func (s *server) List(ctx context.Context, request *oproto.ListRequest) (*oproto.ListResponse, error) {
response := &oproto.ListResponse{
Timer: make([]*oproto.LogMessage, 0),
}
requestVariable := variable.NewFromProto(request.Prefix)
if len(requestVariable.Variable) == 0 {
return nil, fmt.Errorf("No variable specified")
}
// Retrieve all variables and store the names in a map for uniqueness
timer := openinstrument.NewTimer(ctx, "retrieve variables")
vars := make(map[string]*oproto.StreamVariable)
if requestVariable.MinTimestamp == 0 {
// Get the last day
requestVariable.MinTimestamp = -86400000
}
for stream := range s.ds.Reader(ctx, requestVariable) {
if request.MaxVariables == 0 || len(vars) < int(request.MaxVariables) {
vars[variable.ProtoToString(stream.Variable)] = stream.Variable
}
}
timer.Stop()
// Build the response out of the map
timer = openinstrument.NewTimer(ctx, "construct response")
response.Variable = make([]*oproto.StreamVariable, 0)
for _, variable := range vars {
response.Variable = append(response.Variable, variable)
}
response.Success = true
timer.Stop()
log.Printf("Timers: %s", openinstrument.GetLog(ctx))
return response, nil
}
// SplitBlock splits a single block into multiple (usually 2) smaller blocks.
// The new blocks' contents are immedately written to disk and reopened by the Datatstore.
// The old block is removed from disk once the new contents are available.
// This will block writes to a block for the duration of the reindexing.
func (ds *Datastore) SplitBlock(ctx context.Context, block *Block) (*Block, *Block, error) {
defer block.UpdateIndexedCount()
defer block.UpdateLoggedCount()
defer block.UpdateUnloggedCount()
// Compact the block before continuing, to make sure everything is flushed to disk
block.Compact(ctx)
// Work out the optimal split point
splitPoint, leftEndKey := block.GetOptimalSplitPoint(ctx)
if splitPoint == 0 {
return nil, nil, fmt.Errorf("Could not split block %s: not enough streams", block)
}
openinstrument.Logf(ctx, "Calculated optimal split point at %d (%s)", splitPoint, leftEndKey)
// Read in the whole block
leftBlock := NewBlock(ctx, leftEndKey, "", ds.Path)
leftStreams := make(map[string]*oproto.ValueStream)
rightStreams := make(map[string]*oproto.ValueStream)
streams, err := block.GetIndexedStreams(ctx)
if err != nil {
return nil, nil, fmt.Errorf("Couldn't read old block file: %s", err)
}
var leftError, rightError error
func() {
locker := block.LogWriteLocker()
locker.Lock()
defer locker.Unlock()
for stream := range streams {
varName := variable.ProtoToString(stream.Variable)
if varName <= leftBlock.EndKey() {
leftStreams[varName] = stream
} else {
rightStreams[varName] = stream
}
}
wg := new(sync.WaitGroup)
wg.Add(2)
go func() { leftError = leftBlock.Write(ctx, leftStreams); wg.Done() }()
go func() { rightError = block.Write(ctx, rightStreams); wg.Done() }()
wg.Wait()
}()
if leftError != nil {
return nil, nil, fmt.Errorf("Error writing left block: %s", leftError)
}
if rightError != nil {
return nil, nil, fmt.Errorf("Error writing right block: %s", rightError)
}
ds.insertBlock(ctx, leftBlock)
defer leftBlock.UpdateIndexedCount()
defer leftBlock.UpdateLoggedCount()
defer leftBlock.UpdateUnloggedCount()
openinstrument.Logf(ctx, "Split complete, left contains %d streams, right contains %d", len(leftStreams), len(rightStreams))
return leftBlock, block, nil
}
func RunQuery(ctx context.Context, query *oproto.Query, store datastore.ReadableStore) (chan *oproto.ValueStream, error) {
log.Printf("Running query %v", query)
output := make(chan *oproto.ValueStream, 100)
go func() {
defer close(output)
for _, v := range query.Variable {
log.Printf("Returning variable %s", variable.ProtoToString(v))
for stream := range store.Reader(ctx, variable.NewFromProto(v)) {
if stream != nil {
outputStream := &oproto.ValueStream{Variable: stream.Variable}
stv := variable.NewFromProto(stream.Variable)
for _, v := range stream.Value {
if stv.TimestampInsideRange(v.Timestamp) {
outputStream.Value = append(outputStream.Value, v)
}
}
output <- outputStream
}
}
}
for _, child := range query.Aggregation {
log.Printf("Running child aggregation")
input := []*oproto.ValueStream{}
for _, q := range child.Query {
o, err := RunQuery(ctx, q, store)
if err != nil {
return
}
for stream := range o {
input = append(input, stream)
}
}
log.Printf("Child aggregation returned output")
o := []*oproto.ValueStream{}
switch child.Type {
case oproto.StreamAggregation_NONE:
o = input
case oproto.StreamAggregation_MEAN:
o = aggregations.Mean(child.Label, input)
case oproto.StreamAggregation_MAX:
o = aggregations.Max(child.Label, input)
case oproto.StreamAggregation_MIN:
o = aggregations.Min(child.Label, input)
case oproto.StreamAggregation_MEDIAN:
o = aggregations.Median(child.Label, input)
case oproto.StreamAggregation_SUM:
o = aggregations.Sum(child.Label, input)
case oproto.StreamAggregation_STDDEV:
o = aggregations.StdDev(child.Label, input)
case oproto.StreamAggregation_PERCENTILE:
o = aggregations.Percentile(child.Label, child.Param, input)
}
for _, stream := range o {
//log.Println(openinstrument.ProtoText(stream))
output <- stream
}
}
for _, child := range query.Mutation {
log.Printf("Running child mutation")
input, err := RunQuery(ctx, child.Query, store)
if err != nil {
log.Printf("Error in child mutation: %s", err)
return
}
for stream := range input {
var outStream *oproto.ValueStream
switch child.Type {
case oproto.StreamMutation_MEAN:
outStream = mutations.Mean(stream)
case oproto.StreamMutation_INTERPOLATE:
outStream = mutations.Interpolate(uint64(child.Param), stream)
case oproto.StreamMutation_MIN:
outStream = mutations.Min(uint64(child.Param), stream)
case oproto.StreamMutation_MAX:
outStream = mutations.Max(uint64(child.Param), stream)
case oproto.StreamMutation_FIRST:
outStream = mutations.First(uint64(child.Param), stream)
case oproto.StreamMutation_LAST:
outStream = mutations.Last(uint64(child.Param), stream)
case oproto.StreamMutation_RATE:
outStream = mutations.Rate(stream)
case oproto.StreamMutation_ROOT:
outStream = mutations.Root(child.Param, stream)
case oproto.StreamMutation_POWER:
outStream = mutations.Power(child.Param, stream)
case oproto.StreamMutation_ADD:
outStream = mutations.Add(child.Param, stream)
case oproto.StreamMutation_MULTIPLY:
outStream = mutations.Multiply(child.Param, stream)
case oproto.StreamMutation_RATE_SIGNED:
outStream = mutations.SignedRate(stream)
case oproto.StreamMutation_MOVING_AVERAGE:
outStream = mutations.MovingAverage(uint64(child.Param), stream)
}
if outStream == nil {
log.Printf("No stream returned from mutation")
continue
}
outStream.Variable = stream.Variable
output <- outStream
}
}
}()
return output, nil
}
func (block *Block) GetOptimalSplitPoint(ctx context.Context) (int, string) {
keys := make(map[string]int, 0)
func() {
for _, index := range block.Block.Header.Index {
keys[variable.ProtoToString(index.Variable)] = int(index.NumValues)
}
for _, stream := range block.GetLogStreams() {
v := variable.ProtoToString(stream.Variable)
_, ok := keys[v]
if !ok {
keys[v] = len(stream.Value)
} else {
keys[v] += len(stream.Value)
}
}
block.newStreamsLock.RLocker().Lock()
defer block.newStreamsLock.RLocker().Unlock()
for _, stream := range block.NewStreams {
v := variable.ProtoToString(stream.Variable)
_, ok := keys[v]
if !ok {
keys[v] = len(stream.Value)
} else {
keys[v] += len(stream.Value)
}
}
}()
if len(keys) < 2 {
return 0, ""
}
var sortedKeys []string
for key := range keys {
sortedKeys = append(sortedKeys, key)
}
sort.Strings(sortedKeys)
// Look for the split point where there are closest to an equal number of values on both sides
moved := 0
lastDifference := 0
splitPoint := len(sortedKeys) / 2
for {
leftEndKey := sortedKeys[splitPoint-1]
leftCount := 0
rightCount := 0
for _, key := range sortedKeys {
if key <= leftEndKey {
leftCount += keys[key]
} else {
rightCount += keys[key]
}
}
difference := rightCount - leftCount
if splitPoint == 1 || splitPoint == len(keys)-1 {
// Can't move any further
break
}
if difference == 0 {
// Exact split
break
}
if lastDifference != 0 && math.Abs(float64(lastDifference)) < math.Abs(float64(difference)) {
// Last position was closer
if moved < 0 {
// The position directly to the right is the best
splitPoint++
break
} else {
// The position directly to the left is the best
splitPoint--
break
}
}
if difference < 0 {
splitPoint--
moved = -1
lastDifference = difference
} else {
splitPoint++
moved = 1
lastDifference = difference
}
}
return splitPoint, sortedKeys[splitPoint-1]
}
func (block *Block) Compact(ctx context.Context) error {
openinstrument.Logf(ctx, "Compacting block %s\n", block)
startTime := time.Now()
// Update cached number of streams and values
defer block.UpdateIndexedCount()
defer block.UpdateLoggedCount()
defer block.UpdateUnloggedCount()
block.protoLock.Lock()
defer block.protoLock.Unlock()
block.Block.State = oproto.Block_COMPACTING
block.compactStartTime = time.Now()
block.newStreamsLock.Lock()
defer block.newStreamsLock.Unlock()
block.logLock.Lock()
defer block.logLock.Unlock()
streams := make(map[string]*oproto.ValueStream, 0)
// Apply the retention policy during compaction
p, err := store_config.Get().GetRetentionPolicy(ctx)
if err != nil {
return fmt.Errorf("Error getting retention policy from config store: %s", err)
}
policy := retentionpolicy.New(&p)
endKey := ""
appendValues := func(stream *oproto.ValueStream) {
if stream.Variable == nil {
openinstrument.Logf(ctx, "Skipping reading stream that contains no variable")
return
}
varName := variable.ProtoToString(stream.Variable)
out := policy.Apply(stream)
if len(out.Value) == 0 {
//openinstrument.Logf(ctx, "Dropping stream for variable %s", varName)
return
}
outstream, found := streams[varName]
if found {
outstream.Value = append(outstream.Value, stream.Value...)
} else {
streams[varName] = stream
}
if varName > endKey {
endKey = varName
}
}
// Append logged streams
for _, stream := range block.LogStreams {
appendValues(stream)
}
openinstrument.Logf(ctx, "Block log contains %d streams", len(streams))
// Append indexed streams
reader, err := block.GetIndexedStreams(ctx)
if err != nil {
openinstrument.Logf(ctx, "Unable to read block: %s", err)
} else {
for stream := range reader {
appendValues(stream)
}
openinstrument.Logf(ctx, "Compaction read block containing %d streams", len(streams))
}
// Append unlogged (new) streams
if len(block.NewStreams) > 0 {
for _, stream := range block.NewStreams {
appendValues(stream)
}
openinstrument.Logf(ctx, "Compaction added %d unlogged streams, total: %d streams", len(block.NewStreams), len(streams))
}
// The end key may have changed if streams have been dropped
block.Block.EndKey = endKey
if err = block.Write(ctx, streams); err != nil {
openinstrument.Logf(ctx, "Error writing: %s", err)
return err
}
// Delete the log file
os.Remove(block.logFilename())
openinstrument.Logf(ctx, "Deleted log file %s", block.logFilename())
block.LogStreams = make(map[string]*oproto.ValueStream)
block.NewStreams = make([]*oproto.ValueStream, 0)
block.compactEndTime = time.Now()
block.Block.State = oproto.Block_LIVE
block.UpdateSize()
openinstrument.Logf(ctx, "Finished compaction of %s in %v", block, time.Since(startTime))
return nil
}
// Write writes a map of ValueStreams to a single block file on disk.
// The values inside each ValueStream will be sorted and run-length-encoded before writing.
func (block *Block) Write(ctx context.Context, streams map[string]*oproto.ValueStream) error {
// Build the header with a 0-index for each variable
block.Block.Header.Index = []*oproto.BlockHeaderIndex{}
block.Block.Header.EndKey = ""
block.Block.Header.StartTimestamp = 0
block.Block.Header.EndTimestamp = 0
streams = block.RunLengthEncodeStreams(ctx, streams)
for v, stream := range streams {
if v > block.Block.Header.EndKey {
block.Block.Header.EndKey = v
}
// Add this stream to the index
block.Block.Header.Index = append(block.Block.Header.Index, &oproto.BlockHeaderIndex{
Variable: stream.Variable,
Offset: uint64(1), // This must be set non-zero so that the protobuf marshals it to non-empty
MinTimestamp: stream.Value[0].Timestamp,
MaxTimestamp: stream.Value[len(stream.Value)-1].Timestamp,
NumValues: uint32(len(stream.Value)),
})
if block.Block.Header.StartTimestamp == 0 || stream.Value[0].Timestamp < block.Block.Header.StartTimestamp {
block.Block.Header.StartTimestamp = stream.Value[0].Timestamp
}
if stream.Value[len(stream.Value)-1].Timestamp > block.Block.Header.EndTimestamp {
block.Block.Header.EndTimestamp = stream.Value[len(stream.Value)-1].Timestamp
}
}
// Start writing to the new block file
newfilename := fmt.Sprintf("%s.new.%d", block.Filename(), os.Getpid())
newfile, err := protofile.Write(newfilename)
if err != nil {
newfile.Close()
return fmt.Errorf("Can't write to %s: %s\n", newfilename, err)
}
newfile.Write(block.Block.Header)
blockEnd := newfile.Tell()
// Write all the ValueStreams
indexPos := make(map[string]uint64)
var outValues uint32
for _, stream := range streams {
indexPos[variable.ProtoToString(stream.Variable)] = uint64(newfile.Tell())
newfile.Write(stream)
outValues += uint32(len(stream.Value))
}
// Update the offsets in the header, now that all the data has been written
for _, index := range block.Block.Header.Index {
index.Offset = indexPos[variable.ProtoToString(index.Variable)]
}
newfile.WriteAt(0, block.Block.Header)
if blockEnd < newfile.Tell() {
// Sanity check, just in case goprotobuf breaks something again
newfile.Close()
os.Remove(newfilename)
log.Fatalf("Error writing block file %s, header overwrote data", newfilename)
}
newfile.Sync()
newfile.Close()
block.UpdateIndexedCount()
openinstrument.Logf(ctx, "Wrote %d streams / %d values to %s", len(streams), outValues, newfilename)
openinstrument.Logf(ctx, "Block log contains %d stream", len(block.Block.Header.Index))
// Rename the temporary file into place
if err := os.Rename(newfilename, block.Filename()); err != nil {
return fmt.Errorf("Error renaming: %s", err)
}
return nil
}
func (s *MySuite) TestVariableNoLabelsOrBraces(c *C) {
q, err := Parse("/test")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Variable[0]), Equals, "/test")
}
func (s *MySuite) TestVariableTwoLabels(c *C) {
q, err := Parse("/test{x=y,host=a}")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Variable[0]), Equals, "/test{host=a,x=y}")
}
func (s *MySuite) TestLabelWildcard(c *C) {
q, err := Parse("/test{host=*}")
c.Assert(err, IsNil)
query := q.query
c.Check(variable.ProtoToString(query.Variable[0]), Equals, "/test{host=*}")
}
func Query(ctx context.Context, ds *datastore.Datastore, w http.ResponseWriter, req *http.Request) {
query := req.FormValue("q")
showValues := req.FormValue("v") == "1"
type Result struct {
Variable string `json:"name"`
Values [][]interface{} `json:"values"`
}
var duration *time.Duration
requestVariable := variable.NewFromString(query)
if req.FormValue("d") != "" {
d, err := time.ParseDuration(req.FormValue("d"))
if err != nil {
w.WriteHeader(401)
fmt.Fprintf(w, "Invalid duration")
return
}
duration = &d
requestVariable.MinTimestamp = int64(time.Now().UnixNano()-d.Nanoseconds()) / 1000000
}
if query == "" {
w.WriteHeader(401)
fmt.Fprintf(w, "Specify q=")
return
}
results := make([]Result, 0)
for stream := range ds.Reader(ctx, requestVariable) {
r := Result{
Variable: variable.ProtoToString(stream.Variable),
}
if !showValues {
results = append(results, r)
continue
}
r.Values = make([][]interface{}, 0)
if duration == nil {
// Latest value only
if len(stream.Value) > 0 {
v := stream.Value[len(stream.Value)-1]
r.Values = append(r.Values, []interface{}{v.Timestamp, v.DoubleValue})
}
} else {
// All values over a specific time period
for _, v := range stream.Value {
if requestVariable.MinTimestamp == 0 || requestVariable.MinTimestamp > int64(v.Timestamp) {
r.Values = append(r.Values, []interface{}{v.Timestamp, v.DoubleValue})
}
}
}
results = append(results, r)
}
b, err := json.Marshal(results)
if err != nil {
w.WriteHeader(500)
fmt.Fprintf(w, "Couldn't marshal: %s", err)
return
}
w.WriteHeader(200)
w.Write(b)
}
