// WhereFields returns a list of non-"time" fields in the WHERE section of stmt.
func (m *Measurement) WhereFields(stmt *influxql.SelectStatement) []string {
set := newStringSet()
for _, name := range stmt.NamesInWhere() {
if name != "time" && m.HasField(name) {
set.add(name)
}
}
return set.list()
}
// SelectTags returns a list of non-field tags in the SELECT section of stmt.
func (m *Measurement) SelectTags(stmt *influxql.SelectStatement) []string {
set := newStringSet()
for _, name := range stmt.NamesInSelect() {
if !m.HasField(name) && m.HasTagKey(name) {
set.add(name)
}
}
return set.list()
}
// SelectFields returns a list of fields in the SELECT section of stmt.
func (m *Measurement) SelectFields(stmt *influxql.SelectStatement) []string {
set := newStringSet()
for _, name := range stmt.NamesInSelect() {
if m.HasField(name) {
set.add(name)
continue
}
}
return set.list()
}
func (s *Shard) ValidateAggregateFieldsInStatement(measurementName string, stmt *influxql.SelectStatement) error {
s.mu.RLock()
defer s.mu.RUnlock()
validateType := func(aname, fname string, t influxql.DataType) error {
if t != influxql.Float && t != influxql.Integer {
return fmt.Errorf("aggregate '%s' requires numerical field values. Field '%s' is of type %s",
aname, fname, t)
}
return nil
}
m := s.measurementFields[measurementName]
if m == nil {
return fmt.Errorf("measurement not found: %s", measurementName)
}
// If a numerical aggregate is requested, ensure it is only performed on numeric data or on a
// nested aggregate on numeric data.
for _, a := range stmt.FunctionCalls() {
// Check for fields like `derivative(mean(value), 1d)`
var nested *influxql.Call = a
if fn, ok := nested.Args[0].(*influxql.Call); ok {
nested = fn
}
switch lit := nested.Args[0].(type) {
case *influxql.VarRef:
if IsNumeric(nested) {
f := m.Fields[lit.Val]
if err := validateType(a.Name, f.Name, f.Type); err != nil {
return err
}
}
case *influxql.Distinct:
if nested.Name != "count" {
return fmt.Errorf("aggregate call didn't contain a field %s", a.String())
}
if IsNumeric(nested) {
f := m.Fields[lit.Val]
if err := validateType(a.Name, f.Name, f.Type); err != nil {
return err
}
}
default:
return fmt.Errorf("aggregate call didn't contain a field %s", a.String())
}
}
return nil
}
// rewriteSelectStatement performs any necessary query re-writing.
func (lm *SelectMapper) rewriteSelectStatement(stmt *influxql.SelectStatement) (*influxql.SelectStatement, error) {
var err error
// Expand regex expressions in the FROM clause.
sources, err := expandSources(stmt.Sources, lm.shard.index)
if err != nil {
return nil, err
}
stmt.Sources = sources
// Expand wildcards in the fields or GROUP BY.
stmt, err = lm.expandWildcards(stmt)
if err != nil {
return nil, err
}
stmt.RewriteDistinct()
return stmt, nil
}
// DimensionTagSets returns list of tag sets from the GROUP BY section of stmt.
func (m *Measurement) DimensionTagSets(stmt *influxql.SelectStatement) ([]*influxql.TagSet, error) {
_, tagKeys := stmt.Dimensions.Normalize()
for _, n := range stmt.NamesInDimension() {
if m.HasTagKey(n) {
tagKeys = append(tagKeys, n)
}
}
// Get the sorted unique tag sets for this statement.
tagSets, err := m.TagSets(stmt, tagKeys)
if err != nil {
return nil, err
}
return tagSets, nil
}
// RewriteSelectStatement performs any necessary query re-writing.
func (db *DatabaseIndex) RewriteSelectStatement(stmt *influxql.SelectStatement) (*influxql.SelectStatement, error) {
// Expand regex expressions in the FROM clause.
sources, err := db.ExpandSources(stmt.Sources)
if err != nil {
return nil, err
}
stmt.Sources = sources
// Expand wildcards in the fields or GROUP BY.
stmt, err = db.ExpandWildcards(stmt)
if err != nil {
return nil, err
}
stmt.RewriteDistinct()
return stmt, nil
}
// Plan creates an execution plan for the given SelectStatement and returns an Executor.
func (q *QueryExecutor) PlanSelect(stmt *influxql.SelectStatement, chunkSize int) (Executor, error) {
shards := map[uint64]meta.ShardInfo{} // Shards requiring mappers.
// It is important to "stamp" this time so that everywhere we evaluate `now()` in the statement is EXACTLY the same `now`
now := time.Now().UTC()
// Replace instances of "now()" with the current time, and check the resultant times.
stmt.Condition = influxql.Reduce(stmt.Condition, &influxql.NowValuer{Now: now})
tmin, tmax := influxql.TimeRange(stmt.Condition)
if tmax.IsZero() {
tmax = now
}
if tmin.IsZero() {
tmin = time.Unix(0, 0)
}
for _, src := range stmt.Sources {
mm, ok := src.(*influxql.Measurement)
if !ok {
return nil, fmt.Errorf("invalid source type: %#v", src)
}
// Build the set of target shards. Using shard IDs as keys ensures each shard ID
// occurs only once.
shardGroups, err := q.MetaStore.ShardGroupsByTimeRange(mm.Database, mm.RetentionPolicy, tmin, tmax)
if err != nil {
return nil, err
}
for _, g := range shardGroups {
for _, sh := range g.Shards {
shards[sh.ID] = sh
}
}
}
// Build the Mappers, one per shard.
mappers := []Mapper{}
for _, sh := range shards {
m, err := q.ShardMapper.CreateMapper(sh, stmt, chunkSize)
if err != nil {
return nil, err
}
if m == nil {
// No data for this shard, skip it.
continue
}
mappers = append(mappers, m)
}
executor := NewSelectExecutor(stmt, mappers, chunkSize)
return executor, nil
}
// filters walks the where clause of a select statement and returns a map with all series ids
// matching the where clause and any filter expression that should be applied to each
func (m *Measurement) filters(stmt *influxql.SelectStatement) (map[uint64]influxql.Expr, error) {
if stmt.Condition == nil || stmt.OnlyTimeDimensions() {
seriesIdsToExpr := make(map[uint64]influxql.Expr)
for _, id := range m.seriesIDs {
seriesIdsToExpr[id] = nil
}
return seriesIdsToExpr, nil
}
ids, seriesIdsToExpr, err := m.walkWhereForSeriesIds(stmt.Condition)
if err != nil {
return nil, err
}
// Ensure every id is in the map and replace literal true expressions with
// nil so the engine doesn't waste time evaluating them.
for _, id := range ids {
if expr, ok := seriesIdsToExpr[id]; !ok {
seriesIdsToExpr[id] = nil
} else if b, ok := expr.(*influxql.BooleanLiteral); ok && b.Val {
seriesIdsToExpr[id] = nil
}
}
return seriesIdsToExpr, nil
}
// derivativeInterval returns the time interval for the one (and only) derivative func
func derivativeInterval(stmt *influxql.SelectStatement) (time.Duration, error) {
if len(stmt.FunctionCalls()[0].Args) == 2 {
return stmt.FunctionCalls()[0].Args[1].(*influxql.DurationLiteral).Val, nil
}
interval, err := stmt.GroupByInterval()
if err != nil {
return 0, err
}
if interval > 0 {
return interval, nil
}
return time.Second, nil
}
// createTagSetsAndFields returns the tagsets and various fields given a measurement and
// SELECT statement.
func createTagSetsAndFields(m *Measurement, stmt *influxql.SelectStatement) (*tagSetsAndFields, error) {
_, tagKeys, err := stmt.Dimensions.Normalize()
if err != nil {
return nil, err
}
sfs := newStringSet()
sts := newStringSet()
wfs := newStringSet()
// Validate the fields and tags asked for exist and keep track of which are in the select vs the where
for _, n := range stmt.NamesInSelect() {
if m.HasField(n) {
sfs.add(n)
continue
}
if m.HasTagKey(n) {
sts.add(n)
}
}
for _, n := range stmt.NamesInDimension() {
if m.HasTagKey(n) {
tagKeys = append(tagKeys, n)
}
}
for _, n := range stmt.NamesInWhere() {
if n == "time" {
continue
}
if m.HasField(n) {
wfs.add(n)
continue
}
}
// Get the sorted unique tag sets for this statement.
tagSets, err := m.TagSets(stmt, tagKeys)
if err != nil {
return nil, err
}
return &tagSetsAndFields{
tagSets: tagSets,
selectFields: sfs.list(),
selectTags: sts.list(),
whereFields: wfs.list(),
}, nil
}
// expandWildcards returns a new SelectStatement with wildcards expanded
// If only a `SELECT *` is present, without a `GROUP BY *`, both tags and fields expand in the SELECT
// If a `SELECT *` and a `GROUP BY *` are both present, then only fiels are expanded in the `SELECT` and only
// tags are expanded in the `GROUP BY`
func (lm *SelectMapper) expandWildcards(stmt *influxql.SelectStatement) (*influxql.SelectStatement, error) {
// If there are no wildcards in the statement, return it as-is.
if !stmt.HasWildcard() {
return stmt, nil
}
// Use sets to avoid duplicate field names.
fieldSet := map[string]struct{}{}
dimensionSet := map[string]struct{}{}
var fields influxql.Fields
var dimensions influxql.Dimensions
// keep track of where the wildcards are in the select statement
hasFieldWildcard := stmt.HasFieldWildcard()
hasDimensionWildcard := stmt.HasDimensionWildcard()
// Iterate measurements in the FROM clause getting the fields & dimensions for each.
for _, src := range stmt.Sources {
if m, ok := src.(*influxql.Measurement); ok {
// Lookup the measurement in the database.
mm := lm.shard.index.Measurement(m.Name)
if mm == nil {
// This shard have never received data for the measurement. No Mapper
// required.
return stmt, nil
}
// Get the fields for this measurement.
for _, name := range mm.FieldNames() {
if _, ok := fieldSet[name]; ok {
continue
}
fieldSet[name] = struct{}{}
fields = append(fields, &influxql.Field{Expr: &influxql.VarRef{Val: name}})
}
// Add tags to fields if a field wildcard was provided and a dimension wildcard was not.
if hasFieldWildcard && !hasDimensionWildcard {
for _, t := range mm.TagKeys() {
if _, ok := fieldSet[t]; ok {
continue
}
fieldSet[t] = struct{}{}
fields = append(fields, &influxql.Field{Expr: &influxql.VarRef{Val: t}})
}
}
// Get the dimensions for this measurement.
if hasDimensionWildcard {
for _, t := range mm.TagKeys() {
if _, ok := dimensionSet[t]; ok {
continue
}
dimensionSet[t] = struct{}{}
dimensions = append(dimensions, &influxql.Dimension{Expr: &influxql.VarRef{Val: t}})
}
}
}
}
// Return a new SelectStatement with the wild cards rewritten.
return stmt.RewriteWildcards(fields, dimensions), nil
}