func (block *Block) Reader(ctx context.Context, v *variable.Variable) <-chan *oproto.ValueStream { c := make(chan *oproto.ValueStream) maybeReturnStreams := func(stream *oproto.ValueStream) { if stream == nil { return } if len(stream.Value) == 0 { return } if int64(stream.Value[len(stream.Value)-1].Timestamp) < v.MinTimestamp { return } if v.MaxTimestamp != 0 && int64(stream.Value[0].Timestamp) > v.MaxTimestamp { return } cv := variable.NewFromProto(stream.Variable) if !cv.Match(v) { return } c <- stream } go func() { defer close(c) block.logLock.RLock() for _, stream := range block.LogStreams { maybeReturnStreams(stream) } block.logLock.RUnlock() block.newStreamsLock.RLock() for _, stream := range block.NewStreams { maybeReturnStreams(stream) } block.newStreamsLock.RUnlock() for _, index := range block.Block.Header.Index { if index.NumValues == 0 { continue } if int64(index.MaxTimestamp) < v.MinTimestamp { continue } if v.MaxTimestamp != 0 && int64(index.MinTimestamp) > v.MaxTimestamp { continue } cv := variable.NewFromProto(index.Variable) if !cv.Match(v) { continue } stream := block.getIndexedStream(ctx, index) if stream != nil { c <- stream } } }() return c }
func NewAverage(exporter *VariableExporter, v *variable.Variable) *Average { sumVar := variable.NewFromProto(v.AsProto()) sumVar.Variable += "-total-count" totalVar := variable.NewFromProto(v.AsProto()) totalVar.Variable += "-overall-sum" return &Average{ overallSum: NewFloat(exporter, sumVar), totalCount: NewInteger(exporter, totalVar), } }
func MergeBy(streams []*oproto.ValueStream, by string) <-chan []*oproto.ValueStream { c := make(chan []*oproto.ValueStream) go func() { uniqueVars := make(map[string]bool) uniqueLabels := make(map[string]bool) for _, stream := range streams { v := variable.NewFromProto(stream.Variable) uniqueVars[v.Variable] = true labelValue, ok := v.Labels[by] if !ok { uniqueLabels[""] = true } else { uniqueLabels[labelValue] = true } } for varname := range uniqueVars { v := variable.NewFromString(varname) if by == "" { var output []*oproto.ValueStream for _, stream := range streams { testvar := variable.NewFromProto(stream.Variable) if testvar.Variable != v.Variable { continue } output = append(output, stream) } if len(output) > 0 { c <- output } } else { for labelvalue := range uniqueLabels { var output []*oproto.ValueStream for _, stream := range streams { testvar := variable.NewFromProto(stream.Variable) if testvar.Variable != v.Variable { continue } value, ok := testvar.Labels[by] if ok && value == labelvalue { output = append(output, stream) } } if len(output) > 0 { c <- output } } } } close(c) }() return c }
func NewRatio(exporter *VariableExporter, v *variable.Variable) *Ratio { successVar := variable.NewFromProto(v.AsProto()) successVar.Variable += "-success" failureVar := variable.NewFromProto(v.AsProto()) failureVar.Variable += "-failure" totalVar := variable.NewFromProto(v.AsProto()) totalVar.Variable += "-total" return &Ratio{ success: NewInteger(exporter, successVar), failure: NewInteger(exporter, failureVar), total: NewInteger(exporter, totalVar), } }
func (s *MySuite) TestGetStreamForVariable(c *C) { block := NewBlock(context.Background(), "/test/foo", "", s.dataDir) for v := 0; v < 10; v++ { varName := fmt.Sprintf("/test/bar%d", v) block.LogStreams[varName] = &oproto.ValueStream{ Variable: &oproto.StreamVariable{Name: varName}, Value: []*oproto.Value{}, } for i := 0; i < 100; i++ { block.LogStreams[varName].Value = append(block.LogStreams[varName].Value, &oproto.Value{DoubleValue: float64(i)}) } } c.Assert(block.Compact(context.Background()), IsNil) found := false for _, index := range block.Block.Header.Index { cv := variable.NewFromProto(index.Variable) if cv.String() != "/test/bar7" { continue } stream := block.getIndexedStream(context.Background(), index) c.Assert(variable.ProtoToString(stream.Variable), Equals, "/test/bar7") found = true break } c.Assert(found, Equals, true) }
func doesVariableMatch(itemVar *variable.Variable, policyVars []*oproto.StreamVariable) bool { if len(policyVars) == 0 { //log.Printf("Stream variable %s matches default policy", itemVar.String()) return true } for _, v := range policyVars { policyVar := variable.NewFromProto(v) if itemVar.Match(policyVar) { //log.Printf("Stream variable %s matches policy variable %s", itemVar.String(), policyVar.String()) return true } //log.Printf("Stream variable %s doesn't match policy variable %s", itemVar.String(), policyVar.String()) } return false }
// Get the mean of all values at a common time across streams func Mean(by []string, input []*oproto.ValueStream) []*oproto.ValueStream { output := []*oproto.ValueStream{} for _, inputVar := range By(by, input) { streams := []*oproto.ValueStream{} for _, stream := range input { matchVar := variable.NewFromProto(stream.Variable) if matchVar.Match(inputVar) { streams = append(streams, stream) } } stream := &oproto.ValueStream{Variable: inputVar.AsProto()} iPos := make([]int, len(streams)) for { values := []float64{} timestamps := []uint64{} for i := 0; i < len(streams); i++ { if iPos[i] >= len(streams[i].Value) { continue } if streams[i] != nil { values = append(values, streams[i].Value[iPos[i]].DoubleValue) timestamps = append(timestamps, streams[i].Value[iPos[i]].Timestamp) } iPos[i]++ } if len(values) == 0 { break } var total float64 for _, i := range values { total += i } var tsTotal uint64 for _, i := range timestamps { tsTotal += i } stream.Value = append(stream.Value, &oproto.Value{ Timestamp: tsTotal / uint64(len(timestamps)), DoubleValue: total / float64(len(values)), }) } if len(stream.Value) > 0 { output = append(output, stream) } } return output }
func (s *MySuite) TestMergeBy(c *C) { input := []*oproto.ValueStream{ &oproto.ValueStream{ Variable: variable.NewFromString("/test{host=a,other=x}").AsProto(), Value: []*oproto.Value{ &oproto.Value{Timestamp: 1, DoubleValue: 1.0}, &oproto.Value{Timestamp: 4, DoubleValue: 4.0}, }, }, &oproto.ValueStream{ Variable: variable.NewFromString("/test{host=b,other=y}").AsProto(), Value: []*oproto.Value{ &oproto.Value{Timestamp: 2, DoubleValue: 2.0}, &oproto.Value{Timestamp: 5, DoubleValue: 5.0}, }, }, &oproto.ValueStream{ Variable: variable.NewFromString("/test{host=a,other=z}").AsProto(), Value: []*oproto.Value{ &oproto.Value{Timestamp: 3, DoubleValue: 3.0}, &oproto.Value{Timestamp: 6, DoubleValue: 6.0}, }, }, } numSets := 0 for streams := range MergeBy(input, "host") { stv := variable.NewFromProto(streams[0].Variable) if stv.Match(variable.NewFromString("/test{host=a}")) { c.Assert(len(streams), Equals, 2) outCount, err := checkValueOrder(Merge(streams)) if err != nil { c.Error(err) } c.Check(outCount, Equals, 4) } else { c.Check(len(streams), Equals, 1) outCount, err := checkValueOrder(Merge(streams)) if err != nil { c.Error(err) } c.Check(outCount, Equals, 2) } numSets++ } c.Check(numSets, Equals, 2) }
func main() { log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) flag.Parse() if len(flag.Args()) < 1 { log.Fatal("Specify at least one variable to retrieve") } q, err := query.Parse(flag.Arg(0)) if err != nil { log.Fatal("Invalid query:", err) } conn, err := client.NewRpcClient(context.Background(), *connectAddress) if err != nil { log.Fatalf("Error connecting to %s: %s", *connectAddress, err) } defer conn.Close() request := &oproto.GetRequest{ Query: q.AsProto(), MaxValues: uint32(*maxValues), MaxVariables: uint32(*maxVariables), } log.Printf("Sending query: %s", openinstrument.ProtoText(q.AsProto())) c, err := conn.Get(context.Background(), request) if err != nil { log.Fatal(err) } for response := range c { for _, stream := range response.Stream { variable := variable.NewFromProto(stream.Variable).String() for _, value := range stream.Value { fmt.Printf("%s\t%s\t", variable, time.Unix(int64(value.Timestamp/1000), 0)) if value.StringValue == "" { fmt.Printf("%f\n", value.DoubleValue) } else { fmt.Printf("%s\n", value.StringValue) } } } } }
func main() { log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) flag.Parse() if len(flag.Args()) < 1 { log.Fatal("Specify at least one variable to retrieve") } if *connectAddress == "" { log.Fatal("--connect address:port required") } request := &oproto.ListRequest{ Prefix: variable.NewFromString(flag.Arg(0)).AsProto(), MaxVariables: uint32(*maxVariables), } if *maxAge != "" { d, _ := time.ParseDuration(*maxAge) request.Prefix.MinTimestamp = -d.Nanoseconds() / 1000000 } conn, err := client.NewRpcClient(context.Background(), *connectAddress) if err != nil { log.Fatalf("Error connecting to %s: %s", *connectAddress, err) } defer conn.Close() c, err := conn.List(context.Background(), request) if err != nil { log.Fatal(err) } vars := []string{} for response := range c { for _, v := range response.Variable { vars = append(vars, variable.NewFromProto(v).String()) } } sort.Strings(vars) for _, variable := range vars { fmt.Println(variable) } }
func findFirstMatchingPolicy(value *oproto.Value, policies []*oproto.RetentionPolicyItem) *oproto.RetentionPolicyItem { now := openinstrument.NowMs() valueStartAge := now - value.Timestamp if value.EndTimestamp == 0 && value.Timestamp != 0 { value.EndTimestamp = value.Timestamp } valueEndAge := now - value.EndTimestamp for _, item := range policies { if len(item.Variable) == 0 { // No variables supplied, this matches everything return item } for _, i := range item.Variable { // Look for policies that match the variable age v := variable.NewFromProto(i) if v.TimestampInsideRange(valueStartAge) || v.TimestampInsideRange(valueEndAge) { return item } } } return nil }
func (s *server) List(ctx context.Context, request *oproto.ListRequest) (*oproto.ListResponse, error) { response := &oproto.ListResponse{ Timer: make([]*oproto.LogMessage, 0), } requestVariable := variable.NewFromProto(request.Prefix) if len(requestVariable.Variable) == 0 { return nil, fmt.Errorf("No variable specified") } // Retrieve all variables and store the names in a map for uniqueness timer := openinstrument.NewTimer(ctx, "retrieve variables") vars := make(map[string]*oproto.StreamVariable) if requestVariable.MinTimestamp == 0 { // Get the last day requestVariable.MinTimestamp = -86400000 } for stream := range s.ds.Reader(ctx, requestVariable) { if request.MaxVariables == 0 || len(vars) < int(request.MaxVariables) { vars[variable.ProtoToString(stream.Variable)] = stream.Variable } } timer.Stop() // Build the response out of the map timer = openinstrument.NewTimer(ctx, "construct response") response.Variable = make([]*oproto.StreamVariable, 0) for _, variable := range vars { response.Variable = append(response.Variable, variable) } response.Success = true timer.Stop() log.Printf("Timers: %s", openinstrument.GetLog(ctx)) return response, nil }
func RunQuery(ctx context.Context, query *oproto.Query, store datastore.ReadableStore) (chan *oproto.ValueStream, error) { log.Printf("Running query %v", query) output := make(chan *oproto.ValueStream, 100) go func() { defer close(output) for _, v := range query.Variable { log.Printf("Returning variable %s", variable.ProtoToString(v)) for stream := range store.Reader(ctx, variable.NewFromProto(v)) { if stream != nil { outputStream := &oproto.ValueStream{Variable: stream.Variable} stv := variable.NewFromProto(stream.Variable) for _, v := range stream.Value { if stv.TimestampInsideRange(v.Timestamp) { outputStream.Value = append(outputStream.Value, v) } } output <- outputStream } } } for _, child := range query.Aggregation { log.Printf("Running child aggregation") input := []*oproto.ValueStream{} for _, q := range child.Query { o, err := RunQuery(ctx, q, store) if err != nil { return } for stream := range o { input = append(input, stream) } } log.Printf("Child aggregation returned output") o := []*oproto.ValueStream{} switch child.Type { case oproto.StreamAggregation_NONE: o = input case oproto.StreamAggregation_MEAN: o = aggregations.Mean(child.Label, input) case oproto.StreamAggregation_MAX: o = aggregations.Max(child.Label, input) case oproto.StreamAggregation_MIN: o = aggregations.Min(child.Label, input) case oproto.StreamAggregation_MEDIAN: o = aggregations.Median(child.Label, input) case oproto.StreamAggregation_SUM: o = aggregations.Sum(child.Label, input) case oproto.StreamAggregation_STDDEV: o = aggregations.StdDev(child.Label, input) case oproto.StreamAggregation_PERCENTILE: o = aggregations.Percentile(child.Label, child.Param, input) } for _, stream := range o { //log.Println(openinstrument.ProtoText(stream)) output <- stream } } for _, child := range query.Mutation { log.Printf("Running child mutation") input, err := RunQuery(ctx, child.Query, store) if err != nil { log.Printf("Error in child mutation: %s", err) return } for stream := range input { var outStream *oproto.ValueStream switch child.Type { case oproto.StreamMutation_MEAN: outStream = mutations.Mean(stream) case oproto.StreamMutation_INTERPOLATE: outStream = mutations.Interpolate(uint64(child.Param), stream) case oproto.StreamMutation_MIN: outStream = mutations.Min(uint64(child.Param), stream) case oproto.StreamMutation_MAX: outStream = mutations.Max(uint64(child.Param), stream) case oproto.StreamMutation_FIRST: outStream = mutations.First(uint64(child.Param), stream) case oproto.StreamMutation_LAST: outStream = mutations.Last(uint64(child.Param), stream) case oproto.StreamMutation_RATE: outStream = mutations.Rate(stream) case oproto.StreamMutation_ROOT: outStream = mutations.Root(child.Param, stream) case oproto.StreamMutation_POWER: outStream = mutations.Power(child.Param, stream) case oproto.StreamMutation_ADD: outStream = mutations.Add(child.Param, stream) case oproto.StreamMutation_MULTIPLY: outStream = mutations.Multiply(child.Param, stream) case oproto.StreamMutation_RATE_SIGNED: outStream = mutations.SignedRate(stream) case oproto.StreamMutation_MOVING_AVERAGE: outStream = mutations.MovingAverage(uint64(child.Param), stream) } if outStream == nil { log.Printf("No stream returned from mutation") continue } outStream.Variable = stream.Variable output <- outStream } } }() return output, nil }
func (policy *RetentionPolicy) Apply(input *oproto.ValueStream) *oproto.ValueStream { itemVar := variable.NewFromProto(input.Variable) matchingPolicies := make([]*oproto.RetentionPolicyItem, 0) for _, policy := range policy.policy.Policy { if doesVariableMatch(itemVar, policy.Variable) { matchingPolicies = append(matchingPolicies, policy) } } if len(matchingPolicies) == 0 { log.Printf("No matching policies for variable %s", itemVar) } output := &oproto.ValueStream{ Variable: input.Variable, Value: []*oproto.Value{}, } for _, value := range input.Value { // Find first matching policy if policy := findFirstMatchingPolicy(value, matchingPolicies); policy != nil { if policy.Policy == oproto.RetentionPolicyItem_DROP { // Matching policy is DROP, so don't output this value continue } if len(policy.Mutation) == 0 { output.Value = append(output.Value, value) continue } /* // TODO(dparrish): Fix this! for _, mutation := range policy.Mutation { //log.Printf("Applying mutation %s", mutation) var outStream &oproto.ValueStream switch mutation.Type { case oproto.StreamMutation_MEAN: outStream = mutations.Mean(stream) case oproto.StreamMutation_INTERPOLATE: outStream = mutations.Interpolate(uint64(mutation.Param), stream) case oproto.StreamMutation_MIN: outStream = mutations.Min(uint64(mutation.Param), stream) case oproto.StreamMutation_MAX: outStream = mutations.Max(uint64(mutation.Param), stream) case oproto.StreamMutation_FIRST: outStream = mutations.First(uint64(mutation.Param), stream) case oproto.StreamMutation_LAST: outStream = mutations.Last(uint64(mutation.Param), stream) case oproto.StreamMutation_RATE: outStream = mutations.Rate(stream) case oproto.StreamMutation_ROOT: outStream = mutations.Root(mutation.Param, stream) case oproto.StreamMutation_POWER: outStream = mutations.Power(mutation.Param, stream) case oproto.StreamMutation_ADD: outStream = mutations.Add(mutation.Param, stream) case oproto.StreamMutation_MULTIPLY: outStream = mutations.Multiply(mutation.Param, stream) case oproto.StreamMutation_RATE_SIGNED: outStream = mutations.SignedRate(stream) case oproto.StreamMutation_MOVING_AVERAGE: outStream = mutations.MovingAverage(uint64(mutation.Param), stream) } if outStream == nil { log.Printf("No stream returned from mutation") continue } outStream.Variable = stream.Variable } */ } } return output }