func main() {
flag.Parse()
common.SetupLogger()
config := common.LoadConfig()
cassandraConfig := cassandra.CassandraMetricMetadataConfig{
Hosts: config.MetricMetadataConfig.Hosts,
Keyspace: config.MetricMetadataConfig.Keyspace,
}
apiInstance := common.NewMetricMetadataAPI(cassandraConfig)
ruleset, err := util.LoadRules(config.MetricMetadataConfig.ConversionRulesPath)
if err != nil {
//Blah
}
graphite := util.RuleBasedGraphiteConverter{Ruleset: ruleset}
config.Blueflood.GraphiteMetricConverter = &graphite
blueflood := blueflood.NewBlueflood(config.Blueflood)
optimizer := optimize.NewOptimizationConfiguration()
optimizer.EnableMetricMetadataCaching = true
startServer(config.UIConfig, query.ExecutionContext{
MetricMetadataAPI: apiInstance,
TimeseriesStorageAPI: blueflood,
FetchLimit: 1000,
SlotLimit: 5000,
Registry: registry.Default(),
OptimizationConfiguration: optimizer,
})
}
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
if err != nil {
return nil, err
}
hasTimeout := context.Timeout != 0
var cancellable api.Cancellable
if hasTimeout {
cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
} else {
cancellable = api.NewCancellable()
}
r := context.Registry
if r == nil {
r = registry.Default()
}
defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
evaluationContext := function.EvaluationContext{
API: context.API,
FetchLimit: function.NewFetchCounter(context.FetchLimit),
MultiBackend: context.Backend,
Predicate: cmd.predicate,
SampleMethod: cmd.context.SampleMethod,
Timerange: timerange,
Cancellable: cancellable,
Profiler: context.Profiler,
Registry: r,
}
if hasTimeout {
timeout := time.After(context.Timeout)
results := make(chan interface{})
errors := make(chan error)
go func() {
result, err := evaluateExpressions(evaluationContext, cmd.expressions)
if err != nil {
errors <- err
} else {
results <- result
}
}()
select {
case <-timeout:
return nil, fmt.Errorf("Timeout while executing the query.") // timeout.
case result := <-results:
return result, nil
case err := <-errors:
return nil, err
}
} else {
return evaluateExpressions(evaluationContext, cmd.expressions)
}
}
func TestMovingAverage(t *testing.T) {
fakeAPI := mocks.NewFakeMetricMetadataAPI()
fakeAPI.AddPairWithoutGraphite(api.TaggedMetric{"series", api.NewTagSet()})
fakeBackend := movingAverageBackend{}
timerange, err := api.NewTimerange(1200, 1500, 100)
if err != nil {
t.Fatalf(err.Error())
}
expression := &functionExpression{
functionName: "transform.moving_average",
groupBy: []string{},
arguments: []function.Expression{
&metricFetchExpression{"series", api.TruePredicate},
durationExpression{"300ms", 300 * time.Millisecond},
},
}
backend := fakeBackend
result, err := evaluateToSeriesList(expression,
&function.EvaluationContext{
MetricMetadataAPI: fakeAPI,
TimeseriesStorageAPI: backend,
Timerange: timerange,
SampleMethod: api.SampleMean,
FetchLimit: function.NewFetchCounter(1000),
Registry: registry.Default(),
Cancellable: api.NewCancellable(),
OptimizationConfiguration: optimize.NewOptimizationConfiguration(),
})
if err != nil {
t.Errorf(err.Error())
}
expected := []float64{4, 3, 11.0 / 3, 5}
if len(result.Series) != 1 {
t.Fatalf("expected exactly 1 returned series")
}
if len(result.Series[0].Values) != len(expected) {
t.Fatalf("expected exactly %d values in returned series, but got %d", len(expected), len(result.Series[0].Values))
}
const eps = 1e-7
for i := range expected {
if math.Abs(result.Series[0].Values[i]-expected[i]) > eps {
t.Fatalf("expected %+v but got %+v", expected, result.Series[0].Values)
}
}
}
func main() {
flag.Parse()
common.SetupLogger()
//Adding a signal handler to dump goroutines
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGUSR2)
go func() {
for _ = range sigs {
pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
}
}()
config := common.LoadConfig()
cassandraConfig := cassandra.CassandraMetricMetadataConfig{
Hosts: config.MetricMetadataConfig.Hosts,
Keyspace: config.MetricMetadataConfig.Keyspace,
}
apiInstance := common.NewMetricMetadataAPI(cassandraConfig)
ruleset, err := util.LoadRules(config.MetricMetadataConfig.ConversionRulesPath)
if err != nil {
//Blah
}
graphite := util.RuleBasedGraphiteConverter{Ruleset: ruleset}
config.Blueflood.GraphiteMetricConverter = &graphite
blueflood := blueflood.NewBlueflood(config.Blueflood)
optimizer := optimize.NewOptimizationConfiguration()
optimizer.EnableMetricMetadataCaching = true
//Defaults
userConfig := api.UserSpecifiableConfig{
IncludeRawData: false,
}
startServer(config.UIConfig, query.ExecutionContext{
MetricMetadataAPI: apiInstance,
TimeseriesStorageAPI: blueflood,
FetchLimit: 1500,
SlotLimit: 5000,
Registry: registry.Default(),
OptimizationConfiguration: optimizer,
UserSpecifiableConfig: userConfig,
})
}
func TestMovingAverage(t *testing.T) {
fakeAPI := mocks.NewFakeApi()
fakeAPI.AddPair(api.TaggedMetric{"series", api.NewTagSet()}, "series")
fakeBackend := movingAverageBackend{}
timerange, err := api.NewTimerange(1200, 1500, 100)
if err != nil {
t.Fatalf(err.Error())
}
expression := &functionExpression{
functionName: "transform.moving_average",
groupBy: []string{},
arguments: []function.Expression{
&metricFetchExpression{"series", api.TruePredicate},
stringExpression{"300ms"},
},
}
result, err := evaluateToSeriesList(expression,
function.EvaluationContext{
API: fakeAPI,
MultiBackend: backend.NewSequentialMultiBackend(fakeBackend),
Timerange: timerange,
SampleMethod: api.SampleMean,
FetchLimit: function.NewFetchCounter(1000),
Registry: registry.Default(),
})
if err != nil {
t.Errorf(err.Error())
}
expected := []float64{4, 3, 11.0 / 3, 5}
if len(result.Series) != 1 {
t.Fatalf("expected exactly 1 returned series")
}
if len(result.Series[0].Values) != len(expected) {
t.Fatalf("expected exactly %d values in returned series", len(expected))
}
const eps = 1e-7
for i := range expected {
if math.Abs(result.Series[0].Values[i]-expected[i]) > eps {
t.Fatalf("expected %+v but got %+v", expected, result.Series[0].Values)
}
}
}
func Test_ScalarExpression(t *testing.T) {
timerangeA, err := api.NewTimerange(0, 10, 2)
if err != nil {
t.Fatalf("invalid timerange used for testcase")
return
}
for _, test := range []struct {
expr scalarExpression
timerange api.Timerange
expectedSeries []api.Timeseries
}{
{
scalarExpression{5},
timerangeA,
[]api.Timeseries{
api.Timeseries{
Values: []float64{5.0, 5.0, 5.0, 5.0, 5.0, 5.0},
TagSet: api.NewTagSet(),
},
},
},
} {
a := assert.New(t).Contextf("%+v", test)
result, err := evaluateToSeriesList(test.expr, &function.EvaluationContext{
TimeseriesStorageAPI: FakeBackend{},
Timerange: test.timerange,
SampleMethod: api.SampleMean,
FetchLimit: function.NewFetchCounter(1000),
Registry: registry.Default(),
})
if err != nil {
t.Fatalf("failed to convert number into serieslist")
}
a.EqInt(len(result.Series), len(test.expectedSeries))
for i := 0; i < len(result.Series); i++ {
a.Eq(result.Series[i].Values, test.expectedSeries[i].Values)
}
}
}
func main() {
flag.Parse()
common.SetupLogger()
config := common.LoadConfig()
apiInstance := common.NewAPI(config.API)
blueflood := api.ProfilingBackend{
Backend: blueflood.NewBlueflood(config.Blueflood),
}
backend := api.ProfilingMultiBackend{
MultiBackend: backend.NewParallelMultiBackend(blueflood, 20),
}
ui.Main(config.UIConfig, query.ExecutionContext{
API: apiInstance, Backend: backend, FetchLimit: 1000,
Registry: registry.Default(),
})
}
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
if err != nil {
return nil, err
}
slotLimit := context.SlotLimit
defaultLimit := 1000
if slotLimit == 0 {
slotLimit = defaultLimit // the default limit
}
smallestResolution := timerange.Duration() / time.Duration(slotLimit-2)
// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
// (do some algebra)
// (end - start + res) + res <= slots * res
// end - start <= res * (slots - 2)
// so
// res >= (end - start) / (slots - 2)
// Update the timerange by applying the insights of the storage API:
chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(timerange, smallestResolution)
chosenTimerange, err := api.NewSnappedTimerange(timerange.Start(), timerange.End(), int64(chosenResolution/time.Millisecond))
if err != nil {
return nil, err
}
if chosenTimerange.Slots() > slotLimit {
return nil, function.NewLimitError(
"Requested number of data points exceeds the configured limit",
chosenTimerange.Slots(), slotLimit)
}
hasTimeout := context.Timeout != 0
var cancellable api.Cancellable
if hasTimeout {
cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
} else {
cancellable = api.NewCancellable()
}
r := context.Registry
if r == nil {
r = registry.Default()
}
defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
evaluationContext := function.EvaluationContext{
MetricMetadataAPI: context.MetricMetadataAPI,
FetchLimit: function.NewFetchCounter(context.FetchLimit),
TimeseriesStorageAPI: context.TimeseriesStorageAPI,
Predicate: cmd.predicate,
SampleMethod: cmd.context.SampleMethod,
Timerange: timerange,
Cancellable: cancellable,
Registry: r,
Profiler: context.Profiler,
OptimizationConfiguration: context.OptimizationConfiguration,
}
if hasTimeout {
timeout := time.After(context.Timeout)
results := make(chan interface{})
errors := make(chan error)
go func() {
result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
if err != nil {
errors <- err
} else {
results <- result
}
}()
select {
case <-timeout:
return nil, function.NewLimitError("Timeout while executing the query.",
context.Timeout, context.Timeout)
case result := <-results:
return result, nil
case err := <-errors:
return nil, err
}
} else {
values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
if err != nil {
return nil, err
}
lists := make([]api.SeriesList, len(values))
for i := range values {
lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
if err != nil {
return nil, err
}
}
return lists, nil
}
}
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (interface{}, error) {
timerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
if err != nil {
return nil, err
}
slotLimit := context.SlotLimit
defaultLimit := 1000
if slotLimit == 0 {
slotLimit = defaultLimit // the default limit
}
if timerange.Slots() > slotLimit {
return nil, function.NewLimitError(
"Requested number of data points exceeds the configured limit",
timerange.Slots(), slotLimit)
}
hasTimeout := context.Timeout != 0
var cancellable api.Cancellable
if hasTimeout {
cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
} else {
cancellable = api.NewCancellable()
}
r := context.Registry
if r == nil {
r = registry.Default()
}
defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
evaluationContext := function.EvaluationContext{
API: context.API,
FetchLimit: function.NewFetchCounter(context.FetchLimit),
MultiBackend: context.Backend,
Predicate: cmd.predicate,
SampleMethod: cmd.context.SampleMethod,
Timerange: timerange,
Cancellable: cancellable,
Profiler: context.Profiler,
Registry: r,
}
if hasTimeout {
timeout := time.After(context.Timeout)
results := make(chan interface{})
errors := make(chan error)
go func() {
result, err := function.EvaluateMany(evaluationContext, cmd.expressions)
if err != nil {
errors <- err
} else {
results <- result
}
}()
select {
case <-timeout:
return nil, function.NewLimitError("Timeout while executing the query.",
context.Timeout, context.Timeout)
case result := <-results:
return result, nil
case err := <-errors:
return nil, err
}
} else {
values, err := function.EvaluateMany(evaluationContext, cmd.expressions)
if err != nil {
return nil, err
}
lists := make([]api.SeriesList, len(values))
for i := range values {
lists[i], err = values[i].ToSeriesList(evaluationContext.Timerange)
if err != nil {
return nil, err
}
}
return lists, nil
}
}
// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(context ExecutionContext) (CommandResult, error) {
userTimerange, err := api.NewSnappedTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
if err != nil {
return CommandResult{}, err
}
slotLimit := context.SlotLimit
defaultLimit := 1000
if slotLimit == 0 {
slotLimit = defaultLimit // the default limit
}
smallestResolution := userTimerange.Duration() / time.Duration(slotLimit-2)
// ((end + res/2) - (start - res/2)) / res + 1 <= slots // make adjustments for a snap that moves the endpoints
// (do some algebra)
// (end - start + res) + res <= slots * res
// end - start <= res * (slots - 2)
// so
// res >= (end - start) / (slots - 2)
// Update the timerange by applying the insights of the storage API:
chosenResolution := context.TimeseriesStorageAPI.ChooseResolution(userTimerange, smallestResolution)
chosenTimerange, err := api.NewSnappedTimerange(userTimerange.Start(), userTimerange.End(), int64(chosenResolution/time.Millisecond))
if err != nil {
return CommandResult{}, err
}
if chosenTimerange.Slots() > slotLimit {
return CommandResult{}, function.NewLimitError(
"Requested number of data points exceeds the configured limit",
chosenTimerange.Slots(), slotLimit)
}
hasTimeout := context.Timeout != 0
var cancellable api.Cancellable
if hasTimeout {
cancellable = api.NewTimeoutCancellable(time.Now().Add(context.Timeout))
} else {
cancellable = api.NewCancellable()
}
r := context.Registry
if r == nil {
r = registry.Default()
}
defer close(cancellable.Done()) // broadcast the finish - this ensures that the future work is cancelled.
evaluationContext := function.EvaluationContext{
MetricMetadataAPI: context.MetricMetadataAPI,
FetchLimit: function.NewFetchCounter(context.FetchLimit),
TimeseriesStorageAPI: context.TimeseriesStorageAPI,
Predicate: cmd.predicate,
SampleMethod: cmd.context.SampleMethod,
Timerange: chosenTimerange,
Cancellable: cancellable,
Registry: r,
Profiler: context.Profiler,
OptimizationConfiguration: context.OptimizationConfiguration,
EvaluationNotes: []string{},
UserSpecifiableConfig: context.UserSpecifiableConfig,
}
timeout := (<-chan time.Time)(nil)
if hasTimeout {
// A nil channel will just block forever
timeout = time.After(context.Timeout)
}
results := make(chan []function.Value, 1)
errors := make(chan error, 1)
// Goroutines are never garbage collected, so we need to provide capacity so that the send always succeeds.
go func() {
// Evaluate the result, and send it along the goroutines.
result, err := function.EvaluateMany(&evaluationContext, cmd.expressions)
if err != nil {
errors <- err
return
}
results <- result
}()
select {
case <-timeout:
return CommandResult{}, function.NewLimitError("Timeout while executing the query.",
context.Timeout, context.Timeout)
case err := <-errors:
return CommandResult{}, err
case result := <-results:
lists := make([]api.SeriesList, len(result))
for i := range result {
lists[i], err = result[i].ToSeriesList(evaluationContext.Timerange)
if err != nil {
return CommandResult{}, err
}
}
description := map[string][]string{}
for _, list := range lists {
for _, series := range list.Series {
for key, value := range series.TagSet {
description[key] = append(description[key], value)
}
}
}
for key, values := range description {
natural_sort.Sort(values)
filtered := []string{}
for i := range values {
if i == 0 || values[i-1] != values[i] {
filtered = append(filtered, values[i])
}
}
description[key] = filtered
}
return CommandResult{
Body: lists,
Metadata: map[string]interface{}{
"description": description,
"notes": evaluationContext.EvaluationNotes,
},
}, nil
}
}