Esempio n. 1
0
func (block *Block) Reader(ctx context.Context, v *variable.Variable) <-chan *oproto.ValueStream {
	c := make(chan *oproto.ValueStream)

	maybeReturnStreams := func(stream *oproto.ValueStream) {
		if stream == nil {
			return
		}
		if len(stream.Value) == 0 {
			return
		}
		if int64(stream.Value[len(stream.Value)-1].Timestamp) < v.MinTimestamp {
			return
		}
		if v.MaxTimestamp != 0 && int64(stream.Value[0].Timestamp) > v.MaxTimestamp {
			return
		}
		cv := variable.NewFromProto(stream.Variable)
		if !cv.Match(v) {
			return
		}
		c <- stream
	}

	go func() {
		defer close(c)

		block.logLock.RLock()
		for _, stream := range block.LogStreams {
			maybeReturnStreams(stream)
		}
		block.logLock.RUnlock()

		block.newStreamsLock.RLock()
		for _, stream := range block.NewStreams {
			maybeReturnStreams(stream)
		}
		block.newStreamsLock.RUnlock()

		for _, index := range block.Block.Header.Index {
			if index.NumValues == 0 {
				continue
			}
			if int64(index.MaxTimestamp) < v.MinTimestamp {
				continue
			}
			if v.MaxTimestamp != 0 && int64(index.MinTimestamp) > v.MaxTimestamp {
				continue
			}
			cv := variable.NewFromProto(index.Variable)
			if !cv.Match(v) {
				continue
			}
			stream := block.getIndexedStream(ctx, index)
			if stream != nil {
				c <- stream
			}
		}
	}()
	return c
}
Esempio n. 2
0
func NewAverage(exporter *VariableExporter, v *variable.Variable) *Average {
	sumVar := variable.NewFromProto(v.AsProto())
	sumVar.Variable += "-total-count"
	totalVar := variable.NewFromProto(v.AsProto())
	totalVar.Variable += "-overall-sum"
	return &Average{
		overallSum: NewFloat(exporter, sumVar),
		totalCount: NewInteger(exporter, totalVar),
	}
}
Esempio n. 3
0
func MergeBy(streams []*oproto.ValueStream, by string) <-chan []*oproto.ValueStream {
	c := make(chan []*oproto.ValueStream)
	go func() {
		uniqueVars := make(map[string]bool)
		uniqueLabels := make(map[string]bool)
		for _, stream := range streams {
			v := variable.NewFromProto(stream.Variable)
			uniqueVars[v.Variable] = true
			labelValue, ok := v.Labels[by]
			if !ok {
				uniqueLabels[""] = true
			} else {
				uniqueLabels[labelValue] = true
			}
		}
		for varname := range uniqueVars {
			v := variable.NewFromString(varname)
			if by == "" {
				var output []*oproto.ValueStream
				for _, stream := range streams {
					testvar := variable.NewFromProto(stream.Variable)
					if testvar.Variable != v.Variable {
						continue
					}
					output = append(output, stream)
				}
				if len(output) > 0 {
					c <- output
				}
			} else {
				for labelvalue := range uniqueLabels {
					var output []*oproto.ValueStream
					for _, stream := range streams {
						testvar := variable.NewFromProto(stream.Variable)
						if testvar.Variable != v.Variable {
							continue
						}
						value, ok := testvar.Labels[by]
						if ok && value == labelvalue {
							output = append(output, stream)
						}
					}
					if len(output) > 0 {
						c <- output
					}
				}
			}
		}
		close(c)
	}()
	return c
}
Esempio n. 4
0
func NewRatio(exporter *VariableExporter, v *variable.Variable) *Ratio {
	successVar := variable.NewFromProto(v.AsProto())
	successVar.Variable += "-success"
	failureVar := variable.NewFromProto(v.AsProto())
	failureVar.Variable += "-failure"
	totalVar := variable.NewFromProto(v.AsProto())
	totalVar.Variable += "-total"
	return &Ratio{
		success: NewInteger(exporter, successVar),
		failure: NewInteger(exporter, failureVar),
		total:   NewInteger(exporter, totalVar),
	}
}
Esempio n. 5
0
func (s *MySuite) TestGetStreamForVariable(c *C) {
	block := NewBlock(context.Background(), "/test/foo", "", s.dataDir)
	for v := 0; v < 10; v++ {
		varName := fmt.Sprintf("/test/bar%d", v)
		block.LogStreams[varName] = &oproto.ValueStream{
			Variable: &oproto.StreamVariable{Name: varName},
			Value:    []*oproto.Value{},
		}
		for i := 0; i < 100; i++ {
			block.LogStreams[varName].Value = append(block.LogStreams[varName].Value,
				&oproto.Value{DoubleValue: float64(i)})
		}
	}
	c.Assert(block.Compact(context.Background()), IsNil)
	found := false
	for _, index := range block.Block.Header.Index {
		cv := variable.NewFromProto(index.Variable)
		if cv.String() != "/test/bar7" {
			continue
		}
		stream := block.getIndexedStream(context.Background(), index)
		c.Assert(variable.ProtoToString(stream.Variable), Equals, "/test/bar7")
		found = true
		break
	}
	c.Assert(found, Equals, true)
}
Esempio n. 6
0
func doesVariableMatch(itemVar *variable.Variable, policyVars []*oproto.StreamVariable) bool {
	if len(policyVars) == 0 {
		//log.Printf("Stream variable %s matches default policy", itemVar.String())
		return true
	}
	for _, v := range policyVars {
		policyVar := variable.NewFromProto(v)
		if itemVar.Match(policyVar) {
			//log.Printf("Stream variable %s matches policy variable %s", itemVar.String(), policyVar.String())
			return true
		}
		//log.Printf("Stream variable %s doesn't match policy variable %s", itemVar.String(), policyVar.String())
	}
	return false
}
Esempio n. 7
0
// Get the mean of all values at a common time across streams
func Mean(by []string, input []*oproto.ValueStream) []*oproto.ValueStream {
	output := []*oproto.ValueStream{}
	for _, inputVar := range By(by, input) {
		streams := []*oproto.ValueStream{}
		for _, stream := range input {
			matchVar := variable.NewFromProto(stream.Variable)
			if matchVar.Match(inputVar) {
				streams = append(streams, stream)
			}
		}
		stream := &oproto.ValueStream{Variable: inputVar.AsProto()}
		iPos := make([]int, len(streams))
		for {
			values := []float64{}
			timestamps := []uint64{}
			for i := 0; i < len(streams); i++ {
				if iPos[i] >= len(streams[i].Value) {
					continue
				}
				if streams[i] != nil {
					values = append(values, streams[i].Value[iPos[i]].DoubleValue)
					timestamps = append(timestamps, streams[i].Value[iPos[i]].Timestamp)
				}
				iPos[i]++
			}
			if len(values) == 0 {
				break
			}
			var total float64
			for _, i := range values {
				total += i
			}
			var tsTotal uint64
			for _, i := range timestamps {
				tsTotal += i
			}
			stream.Value = append(stream.Value, &oproto.Value{
				Timestamp:   tsTotal / uint64(len(timestamps)),
				DoubleValue: total / float64(len(values)),
			})
		}
		if len(stream.Value) > 0 {
			output = append(output, stream)
		}
	}
	return output
}
func (s *MySuite) TestMergeBy(c *C) {
	input := []*oproto.ValueStream{
		&oproto.ValueStream{
			Variable: variable.NewFromString("/test{host=a,other=x}").AsProto(),
			Value: []*oproto.Value{
				&oproto.Value{Timestamp: 1, DoubleValue: 1.0},
				&oproto.Value{Timestamp: 4, DoubleValue: 4.0},
			},
		},
		&oproto.ValueStream{
			Variable: variable.NewFromString("/test{host=b,other=y}").AsProto(),
			Value: []*oproto.Value{
				&oproto.Value{Timestamp: 2, DoubleValue: 2.0},
				&oproto.Value{Timestamp: 5, DoubleValue: 5.0},
			},
		},
		&oproto.ValueStream{
			Variable: variable.NewFromString("/test{host=a,other=z}").AsProto(),
			Value: []*oproto.Value{
				&oproto.Value{Timestamp: 3, DoubleValue: 3.0},
				&oproto.Value{Timestamp: 6, DoubleValue: 6.0},
			},
		},
	}
	numSets := 0
	for streams := range MergeBy(input, "host") {
		stv := variable.NewFromProto(streams[0].Variable)
		if stv.Match(variable.NewFromString("/test{host=a}")) {
			c.Assert(len(streams), Equals, 2)
			outCount, err := checkValueOrder(Merge(streams))
			if err != nil {
				c.Error(err)
			}
			c.Check(outCount, Equals, 4)
		} else {
			c.Check(len(streams), Equals, 1)
			outCount, err := checkValueOrder(Merge(streams))
			if err != nil {
				c.Error(err)
			}
			c.Check(outCount, Equals, 2)
		}
		numSets++
	}
	c.Check(numSets, Equals, 2)
}
Esempio n. 9
0
func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	flag.Parse()
	if len(flag.Args()) < 1 {
		log.Fatal("Specify at least one variable to retrieve")
	}

	q, err := query.Parse(flag.Arg(0))
	if err != nil {
		log.Fatal("Invalid query:", err)
	}

	conn, err := client.NewRpcClient(context.Background(), *connectAddress)
	if err != nil {
		log.Fatalf("Error connecting to %s: %s", *connectAddress, err)
	}
	defer conn.Close()

	request := &oproto.GetRequest{
		Query:        q.AsProto(),
		MaxValues:    uint32(*maxValues),
		MaxVariables: uint32(*maxVariables),
	}

	log.Printf("Sending query: %s", openinstrument.ProtoText(q.AsProto()))
	c, err := conn.Get(context.Background(), request)
	if err != nil {
		log.Fatal(err)
	}

	for response := range c {
		for _, stream := range response.Stream {
			variable := variable.NewFromProto(stream.Variable).String()
			for _, value := range stream.Value {
				fmt.Printf("%s\t%s\t", variable, time.Unix(int64(value.Timestamp/1000), 0))
				if value.StringValue == "" {
					fmt.Printf("%f\n", value.DoubleValue)
				} else {
					fmt.Printf("%s\n", value.StringValue)
				}
			}
		}
	}

}
Esempio n. 10
0
func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	flag.Parse()
	if len(flag.Args()) < 1 {
		log.Fatal("Specify at least one variable to retrieve")
	}

	if *connectAddress == "" {
		log.Fatal("--connect address:port required")
	}

	request := &oproto.ListRequest{
		Prefix:       variable.NewFromString(flag.Arg(0)).AsProto(),
		MaxVariables: uint32(*maxVariables),
	}
	if *maxAge != "" {
		d, _ := time.ParseDuration(*maxAge)
		request.Prefix.MinTimestamp = -d.Nanoseconds() / 1000000
	}

	conn, err := client.NewRpcClient(context.Background(), *connectAddress)
	if err != nil {
		log.Fatalf("Error connecting to %s: %s", *connectAddress, err)
	}
	defer conn.Close()

	c, err := conn.List(context.Background(), request)
	if err != nil {
		log.Fatal(err)
	}
	vars := []string{}
	for response := range c {
		for _, v := range response.Variable {
			vars = append(vars, variable.NewFromProto(v).String())
		}
	}
	sort.Strings(vars)
	for _, variable := range vars {
		fmt.Println(variable)
	}
}
Esempio n. 11
0
func findFirstMatchingPolicy(value *oproto.Value, policies []*oproto.RetentionPolicyItem) *oproto.RetentionPolicyItem {
	now := openinstrument.NowMs()
	valueStartAge := now - value.Timestamp
	if value.EndTimestamp == 0 && value.Timestamp != 0 {
		value.EndTimestamp = value.Timestamp
	}
	valueEndAge := now - value.EndTimestamp
	for _, item := range policies {
		if len(item.Variable) == 0 {
			// No variables supplied, this matches everything
			return item
		}
		for _, i := range item.Variable {
			// Look for policies that match the variable age
			v := variable.NewFromProto(i)
			if v.TimestampInsideRange(valueStartAge) || v.TimestampInsideRange(valueEndAge) {
				return item
			}
		}
	}
	return nil
}
Esempio n. 12
0
func (s *server) List(ctx context.Context, request *oproto.ListRequest) (*oproto.ListResponse, error) {
	response := &oproto.ListResponse{
		Timer: make([]*oproto.LogMessage, 0),
	}

	requestVariable := variable.NewFromProto(request.Prefix)
	if len(requestVariable.Variable) == 0 {
		return nil, fmt.Errorf("No variable specified")
	}

	// Retrieve all variables and store the names in a map for uniqueness
	timer := openinstrument.NewTimer(ctx, "retrieve variables")
	vars := make(map[string]*oproto.StreamVariable)
	if requestVariable.MinTimestamp == 0 {
		// Get the last day
		requestVariable.MinTimestamp = -86400000
	}
	for stream := range s.ds.Reader(ctx, requestVariable) {
		if request.MaxVariables == 0 || len(vars) < int(request.MaxVariables) {
			vars[variable.ProtoToString(stream.Variable)] = stream.Variable
		}
	}
	timer.Stop()

	// Build the response out of the map
	timer = openinstrument.NewTimer(ctx, "construct response")
	response.Variable = make([]*oproto.StreamVariable, 0)
	for _, variable := range vars {
		response.Variable = append(response.Variable, variable)
	}
	response.Success = true
	timer.Stop()

	log.Printf("Timers: %s", openinstrument.GetLog(ctx))
	return response, nil
}
Esempio n. 13
0
func RunQuery(ctx context.Context, query *oproto.Query, store datastore.ReadableStore) (chan *oproto.ValueStream, error) {
	log.Printf("Running query %v", query)
	output := make(chan *oproto.ValueStream, 100)
	go func() {
		defer close(output)
		for _, v := range query.Variable {
			log.Printf("Returning variable %s", variable.ProtoToString(v))
			for stream := range store.Reader(ctx, variable.NewFromProto(v)) {
				if stream != nil {
					outputStream := &oproto.ValueStream{Variable: stream.Variable}
					stv := variable.NewFromProto(stream.Variable)
					for _, v := range stream.Value {
						if stv.TimestampInsideRange(v.Timestamp) {
							outputStream.Value = append(outputStream.Value, v)
						}
					}
					output <- outputStream
				}
			}
		}
		for _, child := range query.Aggregation {
			log.Printf("Running child aggregation")
			input := []*oproto.ValueStream{}
			for _, q := range child.Query {
				o, err := RunQuery(ctx, q, store)
				if err != nil {
					return
				}
				for stream := range o {
					input = append(input, stream)
				}
			}
			log.Printf("Child aggregation returned output")
			o := []*oproto.ValueStream{}
			switch child.Type {
			case oproto.StreamAggregation_NONE:
				o = input
			case oproto.StreamAggregation_MEAN:
				o = aggregations.Mean(child.Label, input)
			case oproto.StreamAggregation_MAX:
				o = aggregations.Max(child.Label, input)
			case oproto.StreamAggregation_MIN:
				o = aggregations.Min(child.Label, input)
			case oproto.StreamAggregation_MEDIAN:
				o = aggregations.Median(child.Label, input)
			case oproto.StreamAggregation_SUM:
				o = aggregations.Sum(child.Label, input)
			case oproto.StreamAggregation_STDDEV:
				o = aggregations.StdDev(child.Label, input)
			case oproto.StreamAggregation_PERCENTILE:
				o = aggregations.Percentile(child.Label, child.Param, input)
			}
			for _, stream := range o {
				//log.Println(openinstrument.ProtoText(stream))
				output <- stream
			}
		}
		for _, child := range query.Mutation {
			log.Printf("Running child mutation")
			input, err := RunQuery(ctx, child.Query, store)
			if err != nil {
				log.Printf("Error in child mutation: %s", err)
				return
			}
			for stream := range input {
				var outStream *oproto.ValueStream
				switch child.Type {
				case oproto.StreamMutation_MEAN:
					outStream = mutations.Mean(stream)
				case oproto.StreamMutation_INTERPOLATE:
					outStream = mutations.Interpolate(uint64(child.Param), stream)
				case oproto.StreamMutation_MIN:
					outStream = mutations.Min(uint64(child.Param), stream)
				case oproto.StreamMutation_MAX:
					outStream = mutations.Max(uint64(child.Param), stream)
				case oproto.StreamMutation_FIRST:
					outStream = mutations.First(uint64(child.Param), stream)
				case oproto.StreamMutation_LAST:
					outStream = mutations.Last(uint64(child.Param), stream)
				case oproto.StreamMutation_RATE:
					outStream = mutations.Rate(stream)
				case oproto.StreamMutation_ROOT:
					outStream = mutations.Root(child.Param, stream)
				case oproto.StreamMutation_POWER:
					outStream = mutations.Power(child.Param, stream)
				case oproto.StreamMutation_ADD:
					outStream = mutations.Add(child.Param, stream)
				case oproto.StreamMutation_MULTIPLY:
					outStream = mutations.Multiply(child.Param, stream)
				case oproto.StreamMutation_RATE_SIGNED:
					outStream = mutations.SignedRate(stream)
				case oproto.StreamMutation_MOVING_AVERAGE:
					outStream = mutations.MovingAverage(uint64(child.Param), stream)
				}
				if outStream == nil {
					log.Printf("No stream returned from mutation")
					continue
				}
				outStream.Variable = stream.Variable
				output <- outStream
			}
		}
	}()
	return output, nil
}
Esempio n. 14
0
func (policy *RetentionPolicy) Apply(input *oproto.ValueStream) *oproto.ValueStream {
	itemVar := variable.NewFromProto(input.Variable)
	matchingPolicies := make([]*oproto.RetentionPolicyItem, 0)
	for _, policy := range policy.policy.Policy {
		if doesVariableMatch(itemVar, policy.Variable) {
			matchingPolicies = append(matchingPolicies, policy)
		}
	}
	if len(matchingPolicies) == 0 {
		log.Printf("No matching policies for variable %s", itemVar)
	}

	output := &oproto.ValueStream{
		Variable: input.Variable,
		Value:    []*oproto.Value{},
	}

	for _, value := range input.Value {
		// Find first matching policy
		if policy := findFirstMatchingPolicy(value, matchingPolicies); policy != nil {
			if policy.Policy == oproto.RetentionPolicyItem_DROP {
				// Matching policy is DROP, so don't output this value
				continue
			}

			if len(policy.Mutation) == 0 {
				output.Value = append(output.Value, value)
				continue
			}

			/*
				// TODO(dparrish): Fix this!
				for _, mutation := range policy.Mutation {
					//log.Printf("Applying mutation %s", mutation)
					var outStream &oproto.ValueStream
					switch mutation.Type {
					case oproto.StreamMutation_MEAN:
						outStream = mutations.Mean(stream)
					case oproto.StreamMutation_INTERPOLATE:
						outStream = mutations.Interpolate(uint64(mutation.Param), stream)
					case oproto.StreamMutation_MIN:
						outStream = mutations.Min(uint64(mutation.Param), stream)
					case oproto.StreamMutation_MAX:
						outStream = mutations.Max(uint64(mutation.Param), stream)
					case oproto.StreamMutation_FIRST:
						outStream = mutations.First(uint64(mutation.Param), stream)
					case oproto.StreamMutation_LAST:
						outStream = mutations.Last(uint64(mutation.Param), stream)
					case oproto.StreamMutation_RATE:
						outStream = mutations.Rate(stream)
					case oproto.StreamMutation_ROOT:
						outStream = mutations.Root(mutation.Param, stream)
					case oproto.StreamMutation_POWER:
						outStream = mutations.Power(mutation.Param, stream)
					case oproto.StreamMutation_ADD:
						outStream = mutations.Add(mutation.Param, stream)
					case oproto.StreamMutation_MULTIPLY:
						outStream = mutations.Multiply(mutation.Param, stream)
					case oproto.StreamMutation_RATE_SIGNED:
						outStream = mutations.SignedRate(stream)
					case oproto.StreamMutation_MOVING_AVERAGE:
						outStream = mutations.MovingAverage(uint64(mutation.Param), stream)
					}
					if outStream == nil {
						log.Printf("No stream returned from mutation")
						continue
					}
					outStream.Variable = stream.Variable
				}
			*/
		}
	}
	return output
}