func (block *Block) Reader(ctx context.Context, v *variable.Variable) <-chan *oproto.ValueStream {
c := make(chan *oproto.ValueStream)
maybeReturnStreams := func(stream *oproto.ValueStream) {
if stream == nil {
return
}
if len(stream.Value) == 0 {
return
}
if int64(stream.Value[len(stream.Value)-1].Timestamp) < v.MinTimestamp {
return
}
if v.MaxTimestamp != 0 && int64(stream.Value[0].Timestamp) > v.MaxTimestamp {
return
}
cv := variable.NewFromProto(stream.Variable)
if !cv.Match(v) {
return
}
c <- stream
}
go func() {
defer close(c)
block.logLock.RLock()
for _, stream := range block.LogStreams {
maybeReturnStreams(stream)
}
block.logLock.RUnlock()
block.newStreamsLock.RLock()
for _, stream := range block.NewStreams {
maybeReturnStreams(stream)
}
block.newStreamsLock.RUnlock()
for _, index := range block.Block.Header.Index {
if index.NumValues == 0 {
continue
}
if int64(index.MaxTimestamp) < v.MinTimestamp {
continue
}
if v.MaxTimestamp != 0 && int64(index.MinTimestamp) > v.MaxTimestamp {
continue
}
cv := variable.NewFromProto(index.Variable)
if !cv.Match(v) {
continue
}
stream := block.getIndexedStream(ctx, index)
if stream != nil {
c <- stream
}
}
}()
return c
}
func NewAverage(exporter *VariableExporter, v *variable.Variable) *Average {
sumVar := variable.NewFromProto(v.AsProto())
sumVar.Variable += "-total-count"
totalVar := variable.NewFromProto(v.AsProto())
totalVar.Variable += "-overall-sum"
return &Average{
overallSum: NewFloat(exporter, sumVar),
totalCount: NewInteger(exporter, totalVar),
}
}
func MergeBy(streams []*oproto.ValueStream, by string) <-chan []*oproto.ValueStream {
c := make(chan []*oproto.ValueStream)
go func() {
uniqueVars := make(map[string]bool)
uniqueLabels := make(map[string]bool)
for _, stream := range streams {
v := variable.NewFromProto(stream.Variable)
uniqueVars[v.Variable] = true
labelValue, ok := v.Labels[by]
if !ok {
uniqueLabels[""] = true
} else {
uniqueLabels[labelValue] = true
}
}
for varname := range uniqueVars {
v := variable.NewFromString(varname)
if by == "" {
var output []*oproto.ValueStream
for _, stream := range streams {
testvar := variable.NewFromProto(stream.Variable)
if testvar.Variable != v.Variable {
continue
}
output = append(output, stream)
}
if len(output) > 0 {
c <- output
}
} else {
for labelvalue := range uniqueLabels {
var output []*oproto.ValueStream
for _, stream := range streams {
testvar := variable.NewFromProto(stream.Variable)
if testvar.Variable != v.Variable {
continue
}
value, ok := testvar.Labels[by]
if ok && value == labelvalue {
output = append(output, stream)
}
}
if len(output) > 0 {
c <- output
}
}
}
}
close(c)
}()
return c
}
func NewRatio(exporter *VariableExporter, v *variable.Variable) *Ratio {
successVar := variable.NewFromProto(v.AsProto())
successVar.Variable += "-success"
failureVar := variable.NewFromProto(v.AsProto())
failureVar.Variable += "-failure"
totalVar := variable.NewFromProto(v.AsProto())
totalVar.Variable += "-total"
return &Ratio{
success: NewInteger(exporter, successVar),
failure: NewInteger(exporter, failureVar),
total: NewInteger(exporter, totalVar),
}
}
func (s *MySuite) TestGetStreamForVariable(c *C) {
block := NewBlock(context.Background(), "/test/foo", "", s.dataDir)
for v := 0; v < 10; v++ {
varName := fmt.Sprintf("/test/bar%d", v)
block.LogStreams[varName] = &oproto.ValueStream{
Variable: &oproto.StreamVariable{Name: varName},
Value: []*oproto.Value{},
}
for i := 0; i < 100; i++ {
block.LogStreams[varName].Value = append(block.LogStreams[varName].Value,
&oproto.Value{DoubleValue: float64(i)})
}
}
c.Assert(block.Compact(context.Background()), IsNil)
found := false
for _, index := range block.Block.Header.Index {
cv := variable.NewFromProto(index.Variable)
if cv.String() != "/test/bar7" {
continue
}
stream := block.getIndexedStream(context.Background(), index)
c.Assert(variable.ProtoToString(stream.Variable), Equals, "/test/bar7")
found = true
break
}
c.Assert(found, Equals, true)
}
func doesVariableMatch(itemVar *variable.Variable, policyVars []*oproto.StreamVariable) bool {
if len(policyVars) == 0 {
//log.Printf("Stream variable %s matches default policy", itemVar.String())
return true
}
for _, v := range policyVars {
policyVar := variable.NewFromProto(v)
if itemVar.Match(policyVar) {
//log.Printf("Stream variable %s matches policy variable %s", itemVar.String(), policyVar.String())
return true
}
//log.Printf("Stream variable %s doesn't match policy variable %s", itemVar.String(), policyVar.String())
}
return false
}
// Get the mean of all values at a common time across streams
func Mean(by []string, input []*oproto.ValueStream) []*oproto.ValueStream {
output := []*oproto.ValueStream{}
for _, inputVar := range By(by, input) {
streams := []*oproto.ValueStream{}
for _, stream := range input {
matchVar := variable.NewFromProto(stream.Variable)
if matchVar.Match(inputVar) {
streams = append(streams, stream)
}
}
stream := &oproto.ValueStream{Variable: inputVar.AsProto()}
iPos := make([]int, len(streams))
for {
values := []float64{}
timestamps := []uint64{}
for i := 0; i < len(streams); i++ {
if iPos[i] >= len(streams[i].Value) {
continue
}
if streams[i] != nil {
values = append(values, streams[i].Value[iPos[i]].DoubleValue)
timestamps = append(timestamps, streams[i].Value[iPos[i]].Timestamp)
}
iPos[i]++
}
if len(values) == 0 {
break
}
var total float64
for _, i := range values {
total += i
}
var tsTotal uint64
for _, i := range timestamps {
tsTotal += i
}
stream.Value = append(stream.Value, &oproto.Value{
Timestamp: tsTotal / uint64(len(timestamps)),
DoubleValue: total / float64(len(values)),
})
}
if len(stream.Value) > 0 {
output = append(output, stream)
}
}
return output
}
func (s *MySuite) TestMergeBy(c *C) {
input := []*oproto.ValueStream{
&oproto.ValueStream{
Variable: variable.NewFromString("/test{host=a,other=x}").AsProto(),
Value: []*oproto.Value{
&oproto.Value{Timestamp: 1, DoubleValue: 1.0},
&oproto.Value{Timestamp: 4, DoubleValue: 4.0},
},
},
&oproto.ValueStream{
Variable: variable.NewFromString("/test{host=b,other=y}").AsProto(),
Value: []*oproto.Value{
&oproto.Value{Timestamp: 2, DoubleValue: 2.0},
&oproto.Value{Timestamp: 5, DoubleValue: 5.0},
},
},
&oproto.ValueStream{
Variable: variable.NewFromString("/test{host=a,other=z}").AsProto(),
Value: []*oproto.Value{
&oproto.Value{Timestamp: 3, DoubleValue: 3.0},
&oproto.Value{Timestamp: 6, DoubleValue: 6.0},
},
},
}
numSets := 0
for streams := range MergeBy(input, "host") {
stv := variable.NewFromProto(streams[0].Variable)
if stv.Match(variable.NewFromString("/test{host=a}")) {
c.Assert(len(streams), Equals, 2)
outCount, err := checkValueOrder(Merge(streams))
if err != nil {
c.Error(err)
}
c.Check(outCount, Equals, 4)
} else {
c.Check(len(streams), Equals, 1)
outCount, err := checkValueOrder(Merge(streams))
if err != nil {
c.Error(err)
}
c.Check(outCount, Equals, 2)
}
numSets++
}
c.Check(numSets, Equals, 2)
}
func main() {
log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
flag.Parse()
if len(flag.Args()) < 1 {
log.Fatal("Specify at least one variable to retrieve")
}
q, err := query.Parse(flag.Arg(0))
if err != nil {
log.Fatal("Invalid query:", err)
}
conn, err := client.NewRpcClient(context.Background(), *connectAddress)
if err != nil {
log.Fatalf("Error connecting to %s: %s", *connectAddress, err)
}
defer conn.Close()
request := &oproto.GetRequest{
Query: q.AsProto(),
MaxValues: uint32(*maxValues),
MaxVariables: uint32(*maxVariables),
}
log.Printf("Sending query: %s", openinstrument.ProtoText(q.AsProto()))
c, err := conn.Get(context.Background(), request)
if err != nil {
log.Fatal(err)
}
for response := range c {
for _, stream := range response.Stream {
variable := variable.NewFromProto(stream.Variable).String()
for _, value := range stream.Value {
fmt.Printf("%s\t%s\t", variable, time.Unix(int64(value.Timestamp/1000), 0))
if value.StringValue == "" {
fmt.Printf("%f\n", value.DoubleValue)
} else {
fmt.Printf("%s\n", value.StringValue)
}
}
}
}
}
func main() {
log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
flag.Parse()
if len(flag.Args()) < 1 {
log.Fatal("Specify at least one variable to retrieve")
}
if *connectAddress == "" {
log.Fatal("--connect address:port required")
}
request := &oproto.ListRequest{
Prefix: variable.NewFromString(flag.Arg(0)).AsProto(),
MaxVariables: uint32(*maxVariables),
}
if *maxAge != "" {
d, _ := time.ParseDuration(*maxAge)
request.Prefix.MinTimestamp = -d.Nanoseconds() / 1000000
}
conn, err := client.NewRpcClient(context.Background(), *connectAddress)
if err != nil {
log.Fatalf("Error connecting to %s: %s", *connectAddress, err)
}
defer conn.Close()
c, err := conn.List(context.Background(), request)
if err != nil {
log.Fatal(err)
}
vars := []string{}
for response := range c {
for _, v := range response.Variable {
vars = append(vars, variable.NewFromProto(v).String())
}
}
sort.Strings(vars)
for _, variable := range vars {
fmt.Println(variable)
}
}
func findFirstMatchingPolicy(value *oproto.Value, policies []*oproto.RetentionPolicyItem) *oproto.RetentionPolicyItem {
now := openinstrument.NowMs()
valueStartAge := now - value.Timestamp
if value.EndTimestamp == 0 && value.Timestamp != 0 {
value.EndTimestamp = value.Timestamp
}
valueEndAge := now - value.EndTimestamp
for _, item := range policies {
if len(item.Variable) == 0 {
// No variables supplied, this matches everything
return item
}
for _, i := range item.Variable {
// Look for policies that match the variable age
v := variable.NewFromProto(i)
if v.TimestampInsideRange(valueStartAge) || v.TimestampInsideRange(valueEndAge) {
return item
}
}
}
return nil
}
func (s *server) List(ctx context.Context, request *oproto.ListRequest) (*oproto.ListResponse, error) {
response := &oproto.ListResponse{
Timer: make([]*oproto.LogMessage, 0),
}
requestVariable := variable.NewFromProto(request.Prefix)
if len(requestVariable.Variable) == 0 {
return nil, fmt.Errorf("No variable specified")
}
// Retrieve all variables and store the names in a map for uniqueness
timer := openinstrument.NewTimer(ctx, "retrieve variables")
vars := make(map[string]*oproto.StreamVariable)
if requestVariable.MinTimestamp == 0 {
// Get the last day
requestVariable.MinTimestamp = -86400000
}
for stream := range s.ds.Reader(ctx, requestVariable) {
if request.MaxVariables == 0 || len(vars) < int(request.MaxVariables) {
vars[variable.ProtoToString(stream.Variable)] = stream.Variable
}
}
timer.Stop()
// Build the response out of the map
timer = openinstrument.NewTimer(ctx, "construct response")
response.Variable = make([]*oproto.StreamVariable, 0)
for _, variable := range vars {
response.Variable = append(response.Variable, variable)
}
response.Success = true
timer.Stop()
log.Printf("Timers: %s", openinstrument.GetLog(ctx))
return response, nil
}
func RunQuery(ctx context.Context, query *oproto.Query, store datastore.ReadableStore) (chan *oproto.ValueStream, error) {
log.Printf("Running query %v", query)
output := make(chan *oproto.ValueStream, 100)
go func() {
defer close(output)
for _, v := range query.Variable {
log.Printf("Returning variable %s", variable.ProtoToString(v))
for stream := range store.Reader(ctx, variable.NewFromProto(v)) {
if stream != nil {
outputStream := &oproto.ValueStream{Variable: stream.Variable}
stv := variable.NewFromProto(stream.Variable)
for _, v := range stream.Value {
if stv.TimestampInsideRange(v.Timestamp) {
outputStream.Value = append(outputStream.Value, v)
}
}
output <- outputStream
}
}
}
for _, child := range query.Aggregation {
log.Printf("Running child aggregation")
input := []*oproto.ValueStream{}
for _, q := range child.Query {
o, err := RunQuery(ctx, q, store)
if err != nil {
return
}
for stream := range o {
input = append(input, stream)
}
}
log.Printf("Child aggregation returned output")
o := []*oproto.ValueStream{}
switch child.Type {
case oproto.StreamAggregation_NONE:
o = input
case oproto.StreamAggregation_MEAN:
o = aggregations.Mean(child.Label, input)
case oproto.StreamAggregation_MAX:
o = aggregations.Max(child.Label, input)
case oproto.StreamAggregation_MIN:
o = aggregations.Min(child.Label, input)
case oproto.StreamAggregation_MEDIAN:
o = aggregations.Median(child.Label, input)
case oproto.StreamAggregation_SUM:
o = aggregations.Sum(child.Label, input)
case oproto.StreamAggregation_STDDEV:
o = aggregations.StdDev(child.Label, input)
case oproto.StreamAggregation_PERCENTILE:
o = aggregations.Percentile(child.Label, child.Param, input)
}
for _, stream := range o {
//log.Println(openinstrument.ProtoText(stream))
output <- stream
}
}
for _, child := range query.Mutation {
log.Printf("Running child mutation")
input, err := RunQuery(ctx, child.Query, store)
if err != nil {
log.Printf("Error in child mutation: %s", err)
return
}
for stream := range input {
var outStream *oproto.ValueStream
switch child.Type {
case oproto.StreamMutation_MEAN:
outStream = mutations.Mean(stream)
case oproto.StreamMutation_INTERPOLATE:
outStream = mutations.Interpolate(uint64(child.Param), stream)
case oproto.StreamMutation_MIN:
outStream = mutations.Min(uint64(child.Param), stream)
case oproto.StreamMutation_MAX:
outStream = mutations.Max(uint64(child.Param), stream)
case oproto.StreamMutation_FIRST:
outStream = mutations.First(uint64(child.Param), stream)
case oproto.StreamMutation_LAST:
outStream = mutations.Last(uint64(child.Param), stream)
case oproto.StreamMutation_RATE:
outStream = mutations.Rate(stream)
case oproto.StreamMutation_ROOT:
outStream = mutations.Root(child.Param, stream)
case oproto.StreamMutation_POWER:
outStream = mutations.Power(child.Param, stream)
case oproto.StreamMutation_ADD:
outStream = mutations.Add(child.Param, stream)
case oproto.StreamMutation_MULTIPLY:
outStream = mutations.Multiply(child.Param, stream)
case oproto.StreamMutation_RATE_SIGNED:
outStream = mutations.SignedRate(stream)
case oproto.StreamMutation_MOVING_AVERAGE:
outStream = mutations.MovingAverage(uint64(child.Param), stream)
}
if outStream == nil {
log.Printf("No stream returned from mutation")
continue
}
outStream.Variable = stream.Variable
output <- outStream
}
}
}()
return output, nil
}
func (policy *RetentionPolicy) Apply(input *oproto.ValueStream) *oproto.ValueStream {
itemVar := variable.NewFromProto(input.Variable)
matchingPolicies := make([]*oproto.RetentionPolicyItem, 0)
for _, policy := range policy.policy.Policy {
if doesVariableMatch(itemVar, policy.Variable) {
matchingPolicies = append(matchingPolicies, policy)
}
}
if len(matchingPolicies) == 0 {
log.Printf("No matching policies for variable %s", itemVar)
}
output := &oproto.ValueStream{
Variable: input.Variable,
Value: []*oproto.Value{},
}
for _, value := range input.Value {
// Find first matching policy
if policy := findFirstMatchingPolicy(value, matchingPolicies); policy != nil {
if policy.Policy == oproto.RetentionPolicyItem_DROP {
// Matching policy is DROP, so don't output this value
continue
}
if len(policy.Mutation) == 0 {
output.Value = append(output.Value, value)
continue
}
/*
// TODO(dparrish): Fix this!
for _, mutation := range policy.Mutation {
//log.Printf("Applying mutation %s", mutation)
var outStream &oproto.ValueStream
switch mutation.Type {
case oproto.StreamMutation_MEAN:
outStream = mutations.Mean(stream)
case oproto.StreamMutation_INTERPOLATE:
outStream = mutations.Interpolate(uint64(mutation.Param), stream)
case oproto.StreamMutation_MIN:
outStream = mutations.Min(uint64(mutation.Param), stream)
case oproto.StreamMutation_MAX:
outStream = mutations.Max(uint64(mutation.Param), stream)
case oproto.StreamMutation_FIRST:
outStream = mutations.First(uint64(mutation.Param), stream)
case oproto.StreamMutation_LAST:
outStream = mutations.Last(uint64(mutation.Param), stream)
case oproto.StreamMutation_RATE:
outStream = mutations.Rate(stream)
case oproto.StreamMutation_ROOT:
outStream = mutations.Root(mutation.Param, stream)
case oproto.StreamMutation_POWER:
outStream = mutations.Power(mutation.Param, stream)
case oproto.StreamMutation_ADD:
outStream = mutations.Add(mutation.Param, stream)
case oproto.StreamMutation_MULTIPLY:
outStream = mutations.Multiply(mutation.Param, stream)
case oproto.StreamMutation_RATE_SIGNED:
outStream = mutations.SignedRate(stream)
case oproto.StreamMutation_MOVING_AVERAGE:
outStream = mutations.MovingAverage(uint64(mutation.Param), stream)
}
if outStream == nil {
log.Printf("No stream returned from mutation")
continue
}
outStream.Variable = stream.Variable
}
*/
}
}
return output
}