func expandExprWithValues(expr influxql.Expr, keys []string, tagExprs []tagExpr, uniques [][]string, index int) []tagSetExpr { // If we have no more keys left then execute the reduction and return. if index == len(keys) { // Create a map of tag key/values. m := make(map[string]*string, len(keys)) for i, key := range keys { if tagExprs[i].op == influxql.EQ { m[key] = &tagExprs[i].values[0] } else { m[key] = nil } } // TODO: Rewrite full expressions instead of VarRef replacement. // Reduce using the current tag key/value set. // Ignore it if reduces down to "false". e := influxql.Reduce(expr, &tagValuer{tags: m}) if e, ok := e.(*influxql.BooleanLiteral); ok && e.Val == false { return nil } return []tagSetExpr{{values: copyTagExprs(tagExprs), expr: e}} } // Otherwise expand for each possible equality value of the key. var exprs []tagSetExpr for _, v := range uniques[index] { exprs = append(exprs, expandExprWithValues(expr, keys, append(tagExprs, tagExpr{keys[index], []string{v}, influxql.EQ}), uniques, index+1)...) } exprs = append(exprs, expandExprWithValues(expr, keys, append(tagExprs, tagExpr{keys[index], uniques[index], influxql.NEQ}), uniques, index+1)...) return exprs }
// Ensure an expression can be reduced. func TestReduce(t *testing.T) { now := mustParseTime("2000-01-01T00:00:00Z") for i, tt := range []struct { in string out string data Valuer }{ // Number literals. {in: `1 + 2`, out: `3`}, {in: `(foo*2) + ( (4/2) + (3 * 5) - 0.5 )`, out: `(foo * 2) + 16.500`}, {in: `foo(bar(2 + 3), 4)`, out: `foo(bar(5), 4)`}, {in: `4 / 0`, out: `0.000`}, {in: `1 / 2`, out: `0.500`}, {in: `4 = 4`, out: `true`}, {in: `4 <> 4`, out: `false`}, {in: `6 > 4`, out: `true`}, {in: `4 >= 4`, out: `true`}, {in: `4 < 6`, out: `true`}, {in: `4 <= 4`, out: `true`}, {in: `4 AND 5`, out: `4 AND 5`}, // Boolean literals. {in: `true AND false`, out: `false`}, {in: `true OR false`, out: `true`}, {in: `true OR (foo = bar AND 1 > 2)`, out: `true`}, {in: `(foo = bar AND 1 > 2) OR true`, out: `true`}, {in: `false OR (foo = bar AND 1 > 2)`, out: `false`}, {in: `(foo = bar AND 1 > 2) OR false`, out: `false`}, {in: `true = false`, out: `false`}, {in: `true <> false`, out: `true`}, {in: `true + false`, out: `true + false`}, // Time literals. {in: `now() + 2h`, out: `'2000-01-01T02:00:00Z'`, data: map[string]interface{}{"now()": now}}, {in: `now() / 2h`, out: `'2000-01-01T00:00:00Z' / 2h`, data: map[string]interface{}{"now()": now}}, {in: `4µ + now()`, out: `'2000-01-01T00:00:00.000004Z'`, data: map[string]interface{}{"now()": now}}, {in: `now() = now()`, out: `true`, data: map[string]interface{}{"now()": now}}, {in: `now() <> now()`, out: `false`, data: map[string]interface{}{"now()": now}}, {in: `now() < now() + 1h`, out: `true`, data: map[string]interface{}{"now()": now}}, {in: `now() <= now() + 1h`, out: `true`, data: map[string]interface{}{"now()": now}}, {in: `now() >= now() - 1h`, out: `true`, data: map[string]interface{}{"now()": now}}, {in: `now() > now() - 1h`, out: `true`, data: map[string]interface{}{"now()": now}}, {in: `now() - (now() - 60s)`, out: `1m`, data: map[string]interface{}{"now()": now}}, {in: `now() AND now()`, out: `'2000-01-01T00:00:00Z' AND '2000-01-01T00:00:00Z'`, data: map[string]interface{}{"now()": now}}, {in: `now()`, out: `now()`}, // Duration literals. {in: `10m + 1h - 60s`, out: `69m`}, {in: `(10m / 2) * 5`, out: `25m`}, {in: `60s = 1m`, out: `true`}, {in: `60s <> 1m`, out: `false`}, {in: `60s < 1h`, out: `true`}, {in: `60s <= 1h`, out: `true`}, {in: `60s > 12s`, out: `true`}, {in: `60s >= 1m`, out: `true`}, {in: `60s AND 1m`, out: `1m AND 1m`}, {in: `60m / 0`, out: `0s`}, {in: `60m + 50`, out: `1h + 50`}, // String literals. {in: `'foo' + 'bar'`, out: `'foobar'`}, // Variable references. {in: `foo`, out: `'bar'`, data: map[string]interface{}{"foo": "bar"}}, {in: `foo = 'bar'`, out: `true`, data: map[string]interface{}{"foo": "bar"}}, {in: `foo = 'bar'`, out: `false`, data: map[string]interface{}{"foo": nil}}, {in: `foo <> 'bar'`, out: `false`, data: map[string]interface{}{"foo": nil}}, } { // Fold expression. expr := influxql.Reduce(MustParseExpr(tt.in), tt.data) // Compare with expected output. if out := expr.String(); tt.out != out { t.Errorf("%d. %s: unexpected expr:\n\nexp=%s\n\ngot=%s\n\n", i, tt.in, tt.out, out) continue } } }
// mergeSeriesFilters merges two sets of filter expressions and culls series IDs. func mergeSeriesFilters(op influxql.Token, ids SeriesIDs, lfilters, rfilters FilterExprs) (SeriesIDs, FilterExprs) { // Create a map to hold the final set of series filter expressions. filters := make(map[uint64]influxql.Expr, 0) // Resulting list of series IDs var series SeriesIDs // Combining logic: // +==========+==========+==========+=======================+=======================+ // | operator | LHS | RHS | intermediate expr | reduced filter | // +==========+==========+==========+=======================+=======================+ // | | <nil> | <r-expr> | false OR <r-expr> | <r-expr> | // | |----------+----------+-----------------------+-----------------------+ // | OR | <l-expr> | <nil> | <l-expr> OR false | <l-expr> | // | |----------+----------+-----------------------+-----------------------+ // | | <nil> | <nil> | false OR false | false | // | |----------+----------+-----------------------+-----------------------+ // | | <l-expr> | <r-expr> | <l-expr> OR <r-expr> | <l-expr> OR <r-expr> | // +----------+----------+----------+-----------------------+-----------------------+ // | | <nil> | <r-expr> | false AND <r-expr> | false* | // | |----------+----------+-----------------------+-----------------------+ // | AND | <l-expr> | <nil> | <l-expr> AND false | false | // | |----------+----------+-----------------------+-----------------------+ // | | <nil> | <nil> | false AND false | false | // | |----------+----------+-----------------------+-----------------------+ // | | <l-expr> | <r-expr> | <l-expr> AND <r-expr> | <l-expr> AND <r-expr> | // +----------+----------+----------+-----------------------+-----------------------+ // *literal false filters and series IDs should be excluded from the results for _, id := range ids { // Get LHS and RHS filter expressions for this series ID. lfilter, rfilter := lfilters[id], rfilters[id] // Set filter to false if either LHS or RHS expressions were nil. if lfilter == nil { lfilter = &influxql.BooleanLiteral{Val: false} } if rfilter == nil { rfilter = &influxql.BooleanLiteral{Val: false} } // Create the intermediate filter expression for this series ID. be := &influxql.BinaryExpr{ Op: op, LHS: lfilter, RHS: rfilter, } // Reduce the intermediate expression. expr := influxql.Reduce(be, nil) // If the expression reduced to false, exclude this series ID and filter. if b, ok := expr.(*influxql.BooleanLiteral); ok && !b.Val { continue } // Store the series ID and merged filter in the final results. if expr != nil { filters[id] = expr } series = append(series, id) } return series, filters }
func (e *QueryExecutor) executeSelectStatement(stmt *influxql.SelectStatement, chunkSize, statementID int, qid uint64, results chan *influxql.Result, closing <-chan struct{}) error { // It is important to "stamp" this time so that everywhere we evaluate `now()` in the statement is EXACTLY the same `now` now := time.Now().UTC() opt := influxql.SelectOptions{InterruptCh: closing} // Replace instances of "now()" with the current time, and check the resultant times. stmt.Condition = influxql.Reduce(stmt.Condition, &influxql.NowValuer{Now: now}) var err error opt.MinTime, opt.MaxTime, err = influxql.TimeRange(stmt.Condition) if err != nil { return err } if opt.MaxTime.IsZero() { opt.MaxTime = now } if opt.MinTime.IsZero() { opt.MinTime = time.Unix(0, 0) } // Convert DISTINCT into a call. stmt.RewriteDistinct() // Remove "time" from fields list. stmt.RewriteTimeFields() // Create an iterator creator based on the shards in the cluster. ic, err := e.iteratorCreator(stmt, &opt) if err != nil { return err } // Expand regex sources to their actual source names. if stmt.Sources.HasRegex() { sources, err := ic.ExpandSources(stmt.Sources) if err != nil { return err } stmt.Sources = sources } // Rewrite wildcards, if any exist. tmp, err := stmt.RewriteWildcards(ic) if err != nil { return err } stmt = tmp if e.MaxSelectBucketsN > 0 && !stmt.IsRawQuery { interval, err := stmt.GroupByInterval() if err != nil { return err } if interval > 0 { // Determine the start and end time matched to the interval (may not match the actual times). min := opt.MinTime.Truncate(interval) max := opt.MaxTime.Truncate(interval).Add(interval) // Determine the number of buckets by finding the time span and dividing by the interval. buckets := int64(max.Sub(min)) / int64(interval) if int(buckets) > e.MaxSelectBucketsN { return fmt.Errorf("max select bucket count exceeded: %d buckets", buckets) } } } // Create a set of iterators from a selection. itrs, err := influxql.Select(stmt, ic, &opt) if err != nil { return err } if qid != 0 && e.MaxSelectPointN > 0 { monitor := influxql.PointLimitMonitor(itrs, influxql.DefaultStatsInterval, e.MaxSelectPointN) e.QueryManager.MonitorQuery(qid, monitor) } // Generate a row emitter from the iterator set. em := influxql.NewEmitter(itrs, stmt.TimeAscending(), chunkSize) em.Columns = stmt.ColumnNames() em.OmitTime = stmt.OmitTime defer em.Close() // Calculate initial stats across all iterators. stats := influxql.Iterators(itrs).Stats() if e.MaxSelectSeriesN > 0 && stats.SeriesN > e.MaxSelectSeriesN { return fmt.Errorf("max select series count exceeded: %d series", stats.SeriesN) } // Emit rows to the results channel. var writeN int64 var emitted bool for { row := em.Emit() if row == nil { // Check if the query was interrupted while emitting. select { case <-closing: return influxql.ErrQueryInterrupted default: } break } result := &influxql.Result{ StatementID: statementID, Series: []*models.Row{row}, } // Write points back into system for INTO statements. if stmt.Target != nil { if err := e.writeInto(stmt, row); err != nil { return err } writeN += int64(len(row.Values)) continue } // Send results or exit if closing. select { case <-closing: return influxql.ErrQueryInterrupted case results <- result: } emitted = true } // Emit write count if an INTO statement. if stmt.Target != nil { results <- &influxql.Result{ StatementID: statementID, Series: []*models.Row{{ Name: "result", Columns: []string{"time", "written"}, Values: [][]interface{}{{time.Unix(0, 0).UTC(), writeN}}, }}, } return nil } // Always emit at least one result. if !emitted { results <- &influxql.Result{ StatementID: statementID, Series: make([]*models.Row, 0), } } return nil }