// CreateIterator returns an iterator for the data in the shard.
func (s *Shard) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if s.closed() {
return nil, ErrEngineClosed
}
if influxql.Sources(opt.Sources).HasSystemSource() {
return s.createSystemIterator(opt)
}
opt.Sources = influxql.Sources(opt.Sources).Filter(s.database, s.retentionPolicy)
return s.engine.CreateIterator(opt)
}
// CreateIterator returns an iterator for the data in the shard.
func (s *Shard) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if err := s.ready(); err != nil {
return nil, err
}
if influxql.Sources(opt.Sources).HasSystemSource() {
return s.createSystemIterator(opt)
}
opt.Sources = influxql.Sources(opt.Sources).Filter(s.database, s.retentionPolicy)
return s.engine.CreateIterator(opt)
}
// Ensure the query executor can execute a basic query.
func TestQueryExecutor_ExecuteQuery_Select(t *testing.T) {
sh := MustOpenShard()
defer sh.Close()
sh.MustWritePointsString(`
cpu,region=serverA value=1 0
cpu,region=serverA value=2 10
cpu,region=serverB value=3 20
`)
e := NewQueryExecutor()
e.MetaClient.ShardIDsByTimeRangeFn = func(sources influxql.Sources, tmin, tmax time.Time) (a []uint64, err error) {
if !reflect.DeepEqual(sources, influxql.Sources([]influxql.Source{&influxql.Measurement{Database: "db0", RetentionPolicy: "rp0", Name: "cpu"}})) {
t.Fatalf("unexpected sources: %s", spew.Sdump(sources))
} else if tmin.IsZero() {
t.Fatalf("unexpected tmin: %s", tmin)
} else if tmax.IsZero() {
t.Fatalf("unexpected tmax: %s", tmax)
}
return []uint64{100}, nil
}
e.Store.ShardsFn = func(ids []uint64) []*tsdb.Shard {
if !reflect.DeepEqual(ids, []uint64{100}) {
t.Fatalf("unexpected shard ids: %+v", ids)
}
return []*tsdb.Shard{sh.Shard}
}
res := e.MustExecuteQueryString("db0", `SELECT value FROM cpu`)
if s := MustMarshalJSON(res); s != `[{"series":[{"name":"cpu","columns":["time","value"],"values":[["1970-01-01T00:00:00Z",1],["1970-01-01T00:00:10Z",2],["1970-01-01T00:00:20Z",3]]}]}]` {
t.Fatalf("unexpected results: %s", s)
}
}
// SeriesKeys returns a list of series in in all shards in a. If a series
// exists in multiple shards in a, all instances will be combined into a single
// Series by calling Combine on it.
func (a Shards) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) {
if influxql.Sources(opt.Sources).HasSystemSource() {
// Only support a single system source.
if len(opt.Sources) > 1 {
return nil, errors.New("cannot select from multiple system sources")
}
// Meta queries don't need to know the series name and always have a single string.
return []influxql.Series{{Aux: []influxql.DataType{influxql.String}}}, nil
}
seriesMap := make(map[string]influxql.Series)
for _, sh := range a {
series, err := sh.SeriesKeys(opt)
if err != nil {
return nil, err
}
for _, s := range series {
cur, ok := seriesMap[s.ID()]
if ok {
cur.Combine(&s)
} else {
seriesMap[s.ID()] = s
}
}
}
seriesList := make([]influxql.Series, 0, len(seriesMap))
for _, s := range seriesMap {
seriesList = append(seriesList, s)
}
sort.Sort(influxql.SeriesList(seriesList))
return influxql.SeriesList(seriesList), nil
}
// FieldDimensions returns unique sets of fields and dimensions across a list of sources.
func (s *Shard) FieldDimensions(sources influxql.Sources) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) {
if influxql.Sources(sources).HasSystemSource() {
// Only support a single system source.
if len(sources) > 1 {
return nil, nil, errors.New("cannot select from multiple system sources")
}
switch m := sources[0].(type) {
case *influxql.Measurement:
switch m.Name {
case "_fieldKeys":
return map[string]influxql.DataType{
"fieldKey": influxql.String,
"fieldType": influxql.String,
}, nil, nil
case "_measurements":
return map[string]influxql.DataType{"_name": influxql.String}, nil, nil
case "_series":
return map[string]influxql.DataType{"key": influxql.String}, nil, nil
case "_tagKeys":
return map[string]influxql.DataType{"tagKey": influxql.String}, nil, nil
case "_tags":
return map[string]influxql.DataType{
"_tagKey": influxql.String,
"value": influxql.String,
}, nil, nil
}
}
return nil, nil, nil
}
fields = make(map[string]influxql.DataType)
dimensions = make(map[string]struct{})
for _, src := range sources {
switch m := src.(type) {
case *influxql.Measurement:
// Retrieve measurement.
mm := s.index.Measurement(m.Name)
if mm == nil {
continue
}
// Append fields and dimensions.
mf := s.engine.MeasurementFields(m.Name)
if mf != nil {
for name, typ := range mf.FieldSet() {
fields[name] = typ
}
}
for _, key := range mm.TagKeys() {
dimensions[key] = struct{}{}
}
}
}
return
}
// CreateIterator returns an iterator for the data in the shard.
func (s *Shard) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if s.closed() {
return nil, ErrEngineClosed
}
if influxql.Sources(opt.Sources).HasSystemSource() {
return s.createSystemIterator(opt)
}
return s.engine.CreateIterator(opt)
}
func (e *Engine) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) {
seriesList := influxql.SeriesList{}
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return nil, err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
series := influxql.Series{
Name: mm.Name,
Tags: influxql.NewTags(t.Tags),
Aux: make([]influxql.DataType, len(opt.Aux)),
}
// Determine the aux field types.
for _, seriesKey := range t.SeriesKeys {
tags := influxql.NewTags(e.index.TagsForSeries(seriesKey))
for i, field := range opt.Aux {
typ := func() influxql.DataType {
mf := e.measurementFields[mm.Name]
if mf == nil {
return influxql.Unknown
}
f := mf.Field(field)
if f == nil {
return influxql.Unknown
}
return f.Type
}()
if typ == influxql.Unknown {
if v := tags.Value(field); v != "" {
// All tags are strings.
typ = influxql.String
}
}
if typ != influxql.Unknown {
if series.Aux[i] == influxql.Unknown || typ < series.Aux[i] {
series.Aux[i] = typ
}
}
}
}
seriesList = append(seriesList, series)
}
}
return seriesList, nil
}
// planShowMeasurements converts the statement to a SELECT and executes it.
func (q *QueryExecutor) planShowMeasurements(stmt *influxql.ShowMeasurementsStatement, database string, chunkSize int) (Executor, error) {
// Check for time in WHERE clause (not supported).
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW MEASUREMENTS doesn't support time in WHERE clause")
}
condition := stmt.Condition
if source, ok := stmt.Source.(*influxql.Measurement); ok {
var expr influxql.Expr
if source.Regex != nil {
expr = &influxql.BinaryExpr{
Op: influxql.EQREGEX,
LHS: &influxql.VarRef{Val: "name"},
RHS: &influxql.RegexLiteral{Val: source.Regex.Val},
}
} else if source.Name != "" {
expr = &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "name"},
RHS: &influxql.StringLiteral{Val: source.Name},
}
}
// Set condition or "AND" together.
if condition == nil {
condition = expr
} else {
condition = &influxql.BinaryExpr{Op: influxql.AND, LHS: expr, RHS: condition}
}
}
ss := &influxql.SelectStatement{
Fields: influxql.Fields([]*influxql.Field{
{Expr: &influxql.VarRef{Val: "name"}},
}),
Sources: influxql.Sources([]influxql.Source{
&influxql.Measurement{Name: "_measurements"},
}),
Condition: condition,
Offset: stmt.Offset,
Limit: stmt.Limit,
SortFields: stmt.SortFields,
OmitTime: true,
Dedupe: true,
}
// Normalize the statement.
if err := q.normalizeStatement(ss, database); err != nil {
return nil, err
}
return q.PlanSelect(ss, chunkSize)
}
// createVarRefIterator creates an iterator for a variable reference.
func (e *Engine) createVarRefIterator(opt influxql.IteratorOptions) ([]influxql.Iterator, error) {
ref, _ := opt.Expr.(*influxql.VarRef)
var itrs []influxql.Iterator
if err := func() error {
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
// Retrieve the maximum number of fields (without time).
conditionFields := make([]string, len(influxql.ExprNames(opt.Condition)))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
for i, seriesKey := range t.SeriesKeys {
fields := 0
if t.Filters[i] != nil {
// Retrieve non-time fields from this series filter and filter out tags.
for _, f := range influxql.ExprNames(t.Filters[i]) {
if mm.HasField(f) {
conditionFields[fields] = f
fields++
}
}
}
itr, err := e.createVarRefSeriesIterator(ref, mm, seriesKey, t, t.Filters[i], conditionFields[:fields], opt)
if err != nil {
return err
} else if itr == nil {
continue
}
itrs = append(itrs, itr)
}
}
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
return itrs, nil
}
func TestSources_HasSystemSource(t *testing.T) {
sources := influxql.Sources([]influxql.Source{
&influxql.Measurement{
Name: "_measurements",
},
})
ok := sources.HasSystemSource()
if !ok {
t.Errorf("expected to find a system source, found none")
}
sources = influxql.Sources([]influxql.Source{
&influxql.Measurement{
Name: "cpu",
},
})
ok = sources.HasSystemSource()
if ok {
t.Errorf("expected to find no system source, found one")
}
}
func TestSources_Names(t *testing.T) {
sources := influxql.Sources([]influxql.Source{
&influxql.Measurement{
Name: "cpu",
},
&influxql.Measurement{
Name: "mem",
},
})
names := sources.Names()
if names[0] != "cpu" {
t.Errorf("expected cpu, got %s", names[0])
}
if names[1] != "mem" {
t.Errorf("expected mem, got %s", names[1])
}
}
// SeriesKeys returns a list of series in the shard.
func (s *Shard) SeriesKeys(opt influxql.IteratorOptions) (influxql.SeriesList, error) {
if influxql.Sources(opt.Sources).HasSystemSource() {
// Only support a single system source.
if len(opt.Sources) > 1 {
return nil, errors.New("cannot select from multiple system sources")
}
// Meta queries don't need to know the series name and
// always have a single series of strings.
auxFields := make([]influxql.DataType, len(opt.Aux))
for i := range auxFields {
auxFields[i] = influxql.String
}
return []influxql.Series{{Aux: auxFields}}, nil
}
return s.engine.SeriesKeys(opt)
}
// createVarRefIterator creates an iterator for a variable reference.
func (e *Engine) createVarRefIterator(opt influxql.IteratorOptions) ([]influxql.Iterator, error) {
ref, _ := opt.Expr.(*influxql.VarRef)
var itrs []influxql.Iterator
if err := func() error {
mms := tsdb.Measurements(e.index.MeasurementsByName(influxql.Sources(opt.Sources).Names()))
for _, mm := range mms {
// Determine tagsets for this measurement based on dimensions and filters.
tagSets, err := mm.TagSets(opt.Dimensions, opt.Condition)
if err != nil {
return err
}
// Calculate tag sets and apply SLIMIT/SOFFSET.
tagSets = influxql.LimitTagSets(tagSets, opt.SLimit, opt.SOffset)
for _, t := range tagSets {
inputs, err := e.createTagSetIterators(ref, mm, t, opt)
if err != nil {
return err
}
if len(inputs) > 0 && (opt.Limit > 0 || opt.Offset > 0) {
var itr influxql.Iterator
if opt.MergeSorted() {
itr = influxql.NewSortedMergeIterator(inputs, opt)
} else {
itr = influxql.NewMergeIterator(inputs, opt)
}
itrs = append(itrs, newLimitIterator(itr, opt))
} else {
itrs = append(itrs, inputs...)
}
}
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
return itrs, nil
}
// CreateIterator returns a single combined iterator for the shards.
func (a Shards) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error) {
if influxql.Sources(opt.Sources).HasSystemSource() {
return a.createSystemIterator(opt)
}
// Create iterators for each shard.
// Ensure that they are closed if an error occurs.
itrs := make([]influxql.Iterator, 0, len(a))
if err := func() error {
for _, sh := range a {
itr, err := sh.CreateIterator(opt)
if err != nil {
return err
}
itrs = append(itrs, itr)
}
return nil
}(); err != nil {
influxql.Iterators(itrs).Close()
return nil, err
}
// Merge into a single iterator.
if opt.MergeSorted() {
return influxql.NewSortedMergeIterator(itrs, opt), nil
}
itr := influxql.NewMergeIterator(itrs, opt)
if opt.Expr != nil {
if expr, ok := opt.Expr.(*influxql.Call); ok && expr.Name == "count" {
opt.Expr = &influxql.Call{
Name: "sum",
Args: expr.Args,
}
}
}
return influxql.NewCallIterator(itr, opt), nil
}
func (e *StatementExecutor) createIterators(stmt *influxql.SelectStatement, ctx *influxql.ExecutionContext) ([]influxql.Iterator, *influxql.SelectStatement, error) {
// It is important to "stamp" this time so that everywhere we evaluate `now()` in the statement is EXACTLY the same `now`
now := time.Now().UTC()
opt := influxql.SelectOptions{
InterruptCh: ctx.InterruptCh,
NodeID: ctx.ExecutionOptions.NodeID,
MaxSeriesN: e.MaxSelectSeriesN,
}
// Replace instances of "now()" with the current time, and check the resultant times.
nowValuer := influxql.NowValuer{Now: now}
stmt.Condition = influxql.Reduce(stmt.Condition, &nowValuer)
// Replace instances of "now()" with the current time in the dimensions.
for _, d := range stmt.Dimensions {
d.Expr = influxql.Reduce(d.Expr, &nowValuer)
}
var err error
opt.MinTime, opt.MaxTime, err = influxql.TimeRange(stmt.Condition)
if err != nil {
return nil, stmt, err
}
if opt.MaxTime.IsZero() {
// In the case that we're executing a meta query where the user cannot
// specify a time condition, then we expand the default max time
// to the maximum possible value, to ensure that data where all points
// are in the future are returned.
if influxql.Sources(stmt.Sources).HasSystemSource() {
opt.MaxTime = time.Unix(0, influxql.MaxTime).UTC()
} else {
if interval, err := stmt.GroupByInterval(); err != nil {
return nil, stmt, err
} else if interval > 0 {
opt.MaxTime = now
} else {
opt.MaxTime = time.Unix(0, influxql.MaxTime).UTC()
}
}
}
if opt.MinTime.IsZero() {
opt.MinTime = time.Unix(0, influxql.MinTime).UTC()
}
// Convert DISTINCT into a call.
stmt.RewriteDistinct()
// Remove "time" from fields list.
stmt.RewriteTimeFields()
// Rewrite any regex conditions that could make use of the index.
stmt.RewriteRegexConditions()
// Create an iterator creator based on the shards in the cluster.
ic, err := e.iteratorCreator(stmt, &opt)
if err != nil {
return nil, stmt, err
}
// Expand regex sources to their actual source names.
if stmt.Sources.HasRegex() {
sources, err := ic.ExpandSources(stmt.Sources)
if err != nil {
return nil, stmt, err
}
stmt.Sources = sources
}
// Rewrite wildcards, if any exist.
tmp, err := stmt.RewriteFields(ic)
if err != nil {
return nil, stmt, err
}
stmt = tmp
if e.MaxSelectBucketsN > 0 && !stmt.IsRawQuery {
interval, err := stmt.GroupByInterval()
if err != nil {
return nil, stmt, err
}
if interval > 0 {
// Determine the start and end time matched to the interval (may not match the actual times).
min := opt.MinTime.Truncate(interval)
max := opt.MaxTime.Truncate(interval).Add(interval)
// Determine the number of buckets by finding the time span and dividing by the interval.
buckets := int64(max.Sub(min)) / int64(interval)
if int(buckets) > e.MaxSelectBucketsN {
return nil, stmt, fmt.Errorf("max-select-buckets limit exceeded: (%d/%d)", buckets, e.MaxSelectBucketsN)
}
}
}
// Create a set of iterators from a selection.
itrs, err := influxql.Select(stmt, ic, &opt)
if err != nil {
return nil, stmt, err
}
if e.MaxSelectPointN > 0 {
monitor := influxql.PointLimitMonitor(itrs, influxql.DefaultStatsInterval, e.MaxSelectPointN)
ctx.Query.Monitor(monitor)
}
return itrs, stmt, nil
}
func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatement, ctx *influxql.ExecutionContext) error {
// Handle SHOW TAG VALUES separately so it can be optimized.
// https://github.com/influxdata/influxdb/issues/6233
if source, ok := stmt.Sources[0].(*influxql.Measurement); ok && source.Name == "_tags" {
// Use the optimized version only if we have direct access to the database.
if store, ok := e.TSDBStore.(LocalTSDBStore); ok {
return e.executeShowTagValues(stmt, ctx, store)
}
}
// It is important to "stamp" this time so that everywhere we evaluate `now()` in the statement is EXACTLY the same `now`
now := time.Now().UTC()
opt := influxql.SelectOptions{InterruptCh: ctx.InterruptCh}
// Replace instances of "now()" with the current time, and check the resultant times.
nowValuer := influxql.NowValuer{Now: now}
stmt.Condition = influxql.Reduce(stmt.Condition, &nowValuer)
// Replace instances of "now()" with the current time in the dimensions.
for _, d := range stmt.Dimensions {
d.Expr = influxql.Reduce(d.Expr, &nowValuer)
}
var err error
opt.MinTime, opt.MaxTime, err = influxql.TimeRange(stmt.Condition)
if err != nil {
return err
}
if opt.MaxTime.IsZero() {
// In the case that we're executing a meta query where the user cannot
// specify a time condition, then we expand the default max time
// to the maximum possible value, to ensure that data where all points
// are in the future are returned.
if influxql.Sources(stmt.Sources).HasSystemSource() {
opt.MaxTime = time.Unix(0, influxql.MaxTime).UTC()
} else {
opt.MaxTime = now
}
}
if opt.MinTime.IsZero() {
opt.MinTime = time.Unix(0, 0)
}
// Convert DISTINCT into a call.
stmt.RewriteDistinct()
// Remove "time" from fields list.
stmt.RewriteTimeFields()
// Create an iterator creator based on the shards in the cluster.
ic, err := e.iteratorCreator(stmt, &opt)
if err != nil {
return err
}
// Expand regex sources to their actual source names.
if stmt.Sources.HasRegex() {
sources, err := ic.ExpandSources(stmt.Sources)
if err != nil {
return err
}
stmt.Sources = sources
}
// Rewrite wildcards, if any exist.
tmp, err := stmt.RewriteFields(ic)
if err != nil {
return err
}
stmt = tmp
if e.MaxSelectBucketsN > 0 && !stmt.IsRawQuery {
interval, err := stmt.GroupByInterval()
if err != nil {
return err
}
if interval > 0 {
// Determine the start and end time matched to the interval (may not match the actual times).
min := opt.MinTime.Truncate(interval)
max := opt.MaxTime.Truncate(interval).Add(interval)
// Determine the number of buckets by finding the time span and dividing by the interval.
buckets := int64(max.Sub(min)) / int64(interval)
if int(buckets) > e.MaxSelectBucketsN {
return fmt.Errorf("max select bucket count exceeded: %d buckets", buckets)
}
}
}
// Create a set of iterators from a selection.
itrs, err := influxql.Select(stmt, ic, &opt)
if err != nil {
return err
}
if e.MaxSelectPointN > 0 {
monitor := influxql.PointLimitMonitor(itrs, influxql.DefaultStatsInterval, e.MaxSelectPointN)
ctx.Query.Monitor(monitor)
}
// Generate a row emitter from the iterator set.
em := influxql.NewEmitter(itrs, stmt.TimeAscending(), ctx.ChunkSize)
em.Columns = stmt.ColumnNames()
em.OmitTime = stmt.OmitTime
defer em.Close()
// Calculate initial stats across all iterators.
stats := influxql.Iterators(itrs).Stats()
if e.MaxSelectSeriesN > 0 && stats.SeriesN > e.MaxSelectSeriesN {
return fmt.Errorf("max select series count exceeded: %d series", stats.SeriesN)
}
// Emit rows to the results channel.
var writeN int64
var emitted bool
var pointsWriter *BufferedPointsWriter
if stmt.Target != nil {
pointsWriter = NewBufferedPointsWriter(e.PointsWriter, stmt.Target.Measurement.Database, stmt.Target.Measurement.RetentionPolicy, 10000)
}
for {
row, err := em.Emit()
if err != nil {
return err
} else if row == nil {
// Check if the query was interrupted while emitting.
select {
case <-ctx.InterruptCh:
return influxql.ErrQueryInterrupted
default:
}
break
}
// Write points back into system for INTO statements.
if stmt.Target != nil {
if err := e.writeInto(pointsWriter, stmt, row); err != nil {
return err
}
writeN += int64(len(row.Values))
continue
}
result := &influxql.Result{
StatementID: ctx.StatementID,
Series: []*models.Row{row},
}
// Send results or exit if closing.
select {
case <-ctx.InterruptCh:
return influxql.ErrQueryInterrupted
case ctx.Results <- result:
}
emitted = true
}
// Flush remaing points and emit write count if an INTO statement.
if stmt.Target != nil {
if err := pointsWriter.Flush(); err != nil {
return err
}
var messages []*influxql.Message
if ctx.ReadOnly {
messages = append(messages, influxql.ReadOnlyWarning(stmt.String()))
}
ctx.Results <- &influxql.Result{
StatementID: ctx.StatementID,
Messages: messages,
Series: []*models.Row{{
Name: "result",
Columns: []string{"time", "written"},
Values: [][]interface{}{{time.Unix(0, 0).UTC(), writeN}},
}},
}
return nil
}
// Always emit at least one result.
if !emitted {
ctx.Results <- &influxql.Result{
StatementID: ctx.StatementID,
Series: make([]*models.Row, 0),
}
}
return nil
}