// Open opens and initializes the mapper. func (m *RawMapper) Open() error { // Ignore if node has the shard but hasn't written to it yet. if m.shard == nil { return nil } // Rewrite statement. stmt, err := m.shard.index.RewriteSelectStatement(m.stmt) if err != nil { return err } m.stmt = stmt // Set all time-related parameters on the mapper. m.qmin, m.qmax = influxql.TimeRangeAsEpochNano(m.stmt.Condition) // Get a read-only transaction. tx, err := m.shard.engine.Begin(false) if err != nil { return err } m.tx = tx // Collect measurements. mms := Measurements(m.shard.index.MeasurementsByName(m.stmt.SourceNames())) m.selectFields = mms.SelectFields(m.stmt) m.selectTags = mms.SelectTags(m.stmt) m.whereFields = mms.WhereFields(m.stmt) // Open cursors for each measurement. for _, mm := range mms { if err := m.openMeasurement(mm); err != nil { return err } } // Remove cursors if there are not SELECT fields. if len(m.selectFields) == 0 { m.cursors = nil } return nil }
// Open opens and initializes the mapper. func (m *AggregateMapper) Open() error { // Ignore if node has the shard but hasn't written to it yet. if m.shard == nil { return nil } // Rewrite statement. stmt, err := m.shard.index.RewriteSelectStatement(m.stmt) if err != nil { return err } m.stmt = stmt // Set all time-related parameters on the mapper. m.qmin, m.qmax = influxql.TimeRangeAsEpochNano(m.stmt.Condition) if err := m.initializeMapFunctions(); err != nil { return err } // For GROUP BY time queries, limit the number of data points returned by the limit and offset d, err := m.stmt.GroupByInterval() if err != nil { return err } m.intervalSize = d.Nanoseconds() if m.qmin == 0 || m.intervalSize == 0 { m.intervalN = 1 m.intervalSize = m.qmax - m.qmin } else { intervalTop := m.qmax/m.intervalSize*m.intervalSize + m.intervalSize intervalBottom := m.qmin / m.intervalSize * m.intervalSize m.intervalN = int((intervalTop - intervalBottom) / m.intervalSize) } if m.stmt.Limit > 0 || m.stmt.Offset > 0 { // ensure that the offset isn't higher than the number of points we'd get if m.stmt.Offset > m.intervalN { return nil } // Take the lesser of either the pre computed number of GROUP BY buckets that // will be in the result or the limit passed in by the user if m.stmt.Limit < m.intervalN { m.intervalN = m.stmt.Limit } } // If we are exceeding our MaxGroupByPoints error out if m.intervalN > MaxGroupByPoints { return errors.New("too many points in the group by interval. maybe you forgot to specify a where time clause?") } // Ensure that the start time for the results is on the start of the window. m.qminWindow = m.qmin if m.intervalSize > 0 && m.intervalN > 1 { m.qminWindow = m.qminWindow / m.intervalSize * m.intervalSize } // Get a read-only transaction. tx, err := m.shard.engine.Begin(false) if err != nil { return err } m.tx = tx // Collect measurements. mms := Measurements(m.shard.index.MeasurementsByName(m.stmt.SourceNames())) m.selectFields = mms.SelectFields(m.stmt) m.selectTags = mms.SelectTags(m.stmt) m.whereFields = mms.WhereFields(m.stmt) // Open cursors for each measurement. for _, mm := range mms { if err := m.openMeasurement(mm); err != nil { return err } } return nil }
// Open opens the local mapper. func (lm *SelectMapper) Open() error { if lm.remote != nil { return lm.remote.Open() } // This can happen when a shard has been assigned to this node but we have not // written to it so it may not exist yet. if lm.shard == nil { return nil } var err error // Get a read-only transaction. tx, err := lm.shard.engine.Begin(false) if err != nil { return err } lm.tx = tx if s, ok := lm.stmt.(*influxql.SelectStatement); ok { stmt, err := lm.rewriteSelectStatement(s) if err != nil { return err } lm.selectStmt = stmt lm.rawMode = (s.IsRawQuery && !s.HasDistinct()) || s.IsSimpleDerivative() } else { return lm.openMeta() } // Set all time-related parameters on the mapper. lm.queryTMin, lm.queryTMax = influxql.TimeRangeAsEpochNano(lm.selectStmt.Condition) if !lm.rawMode { if err := lm.initializeMapFunctions(); err != nil { return err } // For GROUP BY time queries, limit the number of data points returned by the limit and offset d, err := lm.selectStmt.GroupByInterval() if err != nil { return err } lm.intervalSize = d.Nanoseconds() if lm.queryTMin == 0 || lm.intervalSize == 0 { lm.numIntervals = 1 lm.intervalSize = lm.queryTMax - lm.queryTMin } else { intervalTop := lm.queryTMax/lm.intervalSize*lm.intervalSize + lm.intervalSize intervalBottom := lm.queryTMin / lm.intervalSize * lm.intervalSize lm.numIntervals = int((intervalTop - intervalBottom) / lm.intervalSize) } if lm.selectStmt.Limit > 0 || lm.selectStmt.Offset > 0 { // ensure that the offset isn't higher than the number of points we'd get if lm.selectStmt.Offset > lm.numIntervals { return nil } // Take the lesser of either the pre computed number of GROUP BY buckets that // will be in the result or the limit passed in by the user if lm.selectStmt.Limit < lm.numIntervals { lm.numIntervals = lm.selectStmt.Limit } } // If we are exceeding our MaxGroupByPoints error out if lm.numIntervals > MaxGroupByPoints { return errors.New("too many points in the group by interval. maybe you forgot to specify a where time clause?") } // Ensure that the start time for the results is on the start of the window. lm.queryTMinWindow = lm.queryTMin if lm.intervalSize > 0 && lm.numIntervals > 1 { lm.queryTMinWindow = lm.queryTMinWindow / lm.intervalSize * lm.intervalSize } } selectFields := newStringSet() selectTags := newStringSet() whereFields := newStringSet() // Create the TagSet cursors for the Mapper. for _, src := range lm.selectStmt.Sources { mm, ok := src.(*influxql.Measurement) if !ok { return fmt.Errorf("invalid source type: %#v", src) } m := lm.shard.index.Measurement(mm.Name) if m == nil { // This shard have never received data for the measurement. No Mapper // required. return nil } // Validate that ANY GROUP BY is not a field for thie measurement. if err := m.ValidateGroupBy(lm.selectStmt); err != nil { return err } // Create tagset cursors and determine various field types within SELECT statement. tsf, err := createTagSetsAndFields(m, lm.selectStmt) if err != nil { return err } tagSets := tsf.tagSets selectFields.add(tsf.selectFields...) selectTags.add(tsf.selectTags...) whereFields.add(tsf.whereFields...) // If we only have tags in our select clause we just return if len(selectFields) == 0 && len(selectTags) > 0 { return fmt.Errorf("statement must have at least one field in select clause") } // Validate that any GROUP BY is not on a field if err := m.ValidateGroupBy(lm.selectStmt); err != nil { return err } // SLIMIT and SOFFSET the unique series if lm.selectStmt.SLimit > 0 || lm.selectStmt.SOffset > 0 { if lm.selectStmt.SOffset > len(tagSets) { tagSets = nil } else { if lm.selectStmt.SOffset+lm.selectStmt.SLimit > len(tagSets) { lm.selectStmt.SLimit = len(tagSets) - lm.selectStmt.SOffset } tagSets = tagSets[lm.selectStmt.SOffset : lm.selectStmt.SOffset+lm.selectStmt.SLimit] } } // For aggregate functions, we iterate the cursors in forward order but return the // time bucket results in reverse order. This simplifies the aggregate code in that // they do not need to hand forward and revers semantics. For raw queries, we do need // iterate in reverse order if using order by time desc. direction := Forward if lm.rawMode { direction = lm.timeDirection() } // Create all cursors for reading the data from this shard. for _, t := range tagSets { cursors := []*seriesCursor{} for i, key := range t.SeriesKeys { c := lm.tx.Cursor(key, direction) if c == nil { // No data exists for this key. continue } seriesTags := lm.shard.index.TagsForSeries(key) cm := newSeriesCursor(c, t.Filters[i], seriesTags) cursors = append(cursors, cm) } tsc := newTagSetCursor(m.Name, t.Tags, cursors, lm.shard.FieldCodec(m.Name)) if lm.rawMode { tsc.pointHeap = newPointHeap() //Prime the buffers. for i := 0; i < len(tsc.cursors); i++ { var k int64 var v []byte if direction.Forward() { k, v = tsc.cursors[i].SeekTo(lm.queryTMin) } else { k, v = tsc.cursors[i].SeekTo(lm.queryTMax) } if k == -1 { k, v = tsc.cursors[i].Next() } if k == -1 { continue } p := &pointHeapItem{ timestamp: k, value: v, cursor: tsc.cursors[i], } heap.Push(tsc.pointHeap, p) } } lm.cursors = append(lm.cursors, tsc) } sort.Sort(tagSetCursors(lm.cursors)) } lm.selectFields = selectFields.list() lm.selectTags = selectTags.list() lm.whereFields = whereFields.list() // If the query does not aggregate, then at least 1 SELECT field should be present. if lm.rawMode && len(lm.selectFields) == 0 { // None of the SELECT fields exist in this data. Wipe out all tagset cursors. lm.cursors = nil } return nil }
// Open opens the raw mapper. func (rm *RawMapper) Open() error { // Get a read-only transaction. tx, err := rm.shard.DB().Begin(false) if err != nil { return err } rm.tx = tx // Set all time-related parameters on the mapper. rm.queryTMin, rm.queryTMax = influxql.TimeRangeAsEpochNano(rm.stmt.Condition) // Create the TagSet cursors for the Mapper. for _, src := range rm.stmt.Sources { mm, ok := src.(*influxql.Measurement) if !ok { return fmt.Errorf("invalid source type: %#v", src) } m := rm.shard.index.Measurement(mm.Name) if m == nil { // This shard have never received data for the measurement. No Mapper // required. return nil } // Create tagset cursors and determine various field types within SELECT statement. tsf, err := createTagSetsAndFields(m, rm.stmt) if err != nil { return err } tagSets := tsf.tagSets rm.selectFields = tsf.selectFields rm.selectTags = tsf.selectTags rm.whereFields = tsf.whereFields if len(rm.selectFields) == 0 { return fmt.Errorf("select statement must include at least one field") } // SLIMIT and SOFFSET the unique series if rm.stmt.SLimit > 0 || rm.stmt.SOffset > 0 { if rm.stmt.SOffset > len(tagSets) { tagSets = nil } else { if rm.stmt.SOffset+rm.stmt.SLimit > len(tagSets) { rm.stmt.SLimit = len(tagSets) - rm.stmt.SOffset } tagSets = tagSets[rm.stmt.SOffset : rm.stmt.SOffset+rm.stmt.SLimit] } } // Create all cursors for reading the data from this shard. for _, t := range tagSets { cursors := []*seriesCursor{} for i, key := range t.SeriesKeys { c := createCursorForSeries(rm.tx, rm.shard, key) if c == nil { // No data exists for this key. continue } cm := newSeriesCursor(c, t.Filters[i]) cursors = append(cursors, cm) } tsc := newTagSetCursor(m.Name, t.Tags, cursors, rm.shard.FieldCodec(m.Name)) // Prime the buffers. for i := 0; i < len(tsc.cursors); i++ { k, v := tsc.cursors[i].SeekTo(rm.queryTMin) tsc.keyBuffer[i] = k tsc.valueBuffer[i] = v } rm.cursors = append(rm.cursors, tsc) } sort.Sort(tagSetCursors(rm.cursors)) } return nil }
// Open opens the aggregate mapper. func (am *AggMapper) Open() error { var err error // Get a read-only transaction. tx, err := am.shard.DB().Begin(false) if err != nil { return err } am.tx = tx // Set up each mapping function for this statement. aggregates := am.stmt.FunctionCalls() am.mapFuncs = make([]influxql.MapFunc, len(aggregates)) am.fieldNames = make([]string, len(am.mapFuncs)) for i, c := range aggregates { am.mapFuncs[i], err = influxql.InitializeMapFunc(c) if err != nil { return err } // Check for calls like `derivative(mean(value), 1d)` var nested *influxql.Call = c if fn, ok := c.Args[0].(*influxql.Call); ok { nested = fn } switch lit := nested.Args[0].(type) { case *influxql.VarRef: am.fieldNames[i] = lit.Val case *influxql.Distinct: if c.Name != "count" { return fmt.Errorf("aggregate call didn't contain a field %s", c.String()) } am.fieldNames[i] = lit.Val default: return fmt.Errorf("aggregate call didn't contain a field %s", c.String()) } } // Set all time-related parameters on the mapper. am.queryTMin, am.queryTMax = influxql.TimeRangeAsEpochNano(am.stmt.Condition) // For GROUP BY time queries, limit the number of data points returned by the limit and offset d, err := am.stmt.GroupByInterval() if err != nil { return err } am.intervalSize = d.Nanoseconds() if am.queryTMin == 0 || am.intervalSize == 0 { am.numIntervals = 1 am.intervalSize = am.queryTMax - am.queryTMin } else { intervalTop := am.queryTMax/am.intervalSize*am.intervalSize + am.intervalSize intervalBottom := am.queryTMin / am.intervalSize * am.intervalSize am.numIntervals = int((intervalTop - intervalBottom) / am.intervalSize) } if am.stmt.Limit > 0 || am.stmt.Offset > 0 { // ensure that the offset isn't higher than the number of points we'd get if am.stmt.Offset > am.numIntervals { return nil } // Take the lesser of either the pre computed number of GROUP BY buckets that // will be in the result or the limit passed in by the user if am.stmt.Limit < am.numIntervals { am.numIntervals = am.stmt.Limit } } // If we are exceeding our MaxGroupByPoints error out if am.numIntervals > MaxGroupByPoints { return errors.New("too many points in the group by interval. maybe you forgot to specify a where time clause?") } // Ensure that the start time for the results is on the start of the window. am.queryTMinWindow = am.queryTMin if am.intervalSize > 0 && am.numIntervals > 1 { am.queryTMinWindow = am.queryTMinWindow / am.intervalSize * am.intervalSize } // Create the TagSet cursors for the Mapper. for _, src := range am.stmt.Sources { mm, ok := src.(*influxql.Measurement) if !ok { return fmt.Errorf("invalid source type: %#v", src) } m := am.shard.index.Measurement(mm.Name) if m == nil { // This shard have never received data for the measurement. No Mapper // required. return nil } // Create tagset cursors and determine various field types within SELECT statement. tsf, err := createTagSetsAndFields(m, am.stmt) if err != nil { return err } tagSets := tsf.tagSets am.selectFields = tsf.selectFields am.selectTags = tsf.selectTags am.whereFields = tsf.whereFields // Validate that group by is not a field if err := m.ValidateGroupBy(am.stmt); err != nil { return err } // SLIMIT and SOFFSET the unique series if am.stmt.SLimit > 0 || am.stmt.SOffset > 0 { if am.stmt.SOffset > len(tagSets) { tagSets = nil } else { if am.stmt.SOffset+am.stmt.SLimit > len(tagSets) { am.stmt.SLimit = len(tagSets) - am.stmt.SOffset } tagSets = tagSets[am.stmt.SOffset : am.stmt.SOffset+am.stmt.SLimit] } } // Create all cursors for reading the data from this shard. for _, t := range tagSets { cursors := []*seriesCursor{} for i, key := range t.SeriesKeys { c := createCursorForSeries(am.tx, am.shard, key) if c == nil { // No data exists for this key. continue } cm := newSeriesCursor(c, t.Filters[i]) cursors = append(cursors, cm) } tsc := newTagSetCursor(m.Name, t.Tags, cursors, am.shard.FieldCodec(m.Name)) am.cursors = append(am.cursors, tsc) } sort.Sort(tagSetCursors(am.cursors)) } return nil }
// Open opens the local mapper. func (lm *LocalMapper) Open() error { var err error // Get a read-only transaction. tx, err := lm.shard.engine.Begin(false) if err != nil { return err } lm.tx = tx if lm.selectStmt == nil { return lm.openMeta() } // Set all time-related parameters on the mapper. lm.queryTMin, lm.queryTMax = influxql.TimeRangeAsEpochNano(lm.selectStmt.Condition) if !lm.rawMode { if err := lm.initializeMapFunctions(); err != nil { return err } // For GROUP BY time queries, limit the number of data points returned by the limit and offset d, err := lm.selectStmt.GroupByInterval() if err != nil { return err } lm.intervalSize = d.Nanoseconds() if lm.queryTMin == 0 || lm.intervalSize == 0 { lm.numIntervals = 1 lm.intervalSize = lm.queryTMax - lm.queryTMin } else { intervalTop := lm.queryTMax/lm.intervalSize*lm.intervalSize + lm.intervalSize intervalBottom := lm.queryTMin / lm.intervalSize * lm.intervalSize lm.numIntervals = int((intervalTop - intervalBottom) / lm.intervalSize) } if lm.selectStmt.Limit > 0 || lm.selectStmt.Offset > 0 { // ensure that the offset isn't higher than the number of points we'd get if lm.selectStmt.Offset > lm.numIntervals { return nil } // Take the lesser of either the pre computed number of GROUP BY buckets that // will be in the result or the limit passed in by the user if lm.selectStmt.Limit < lm.numIntervals { lm.numIntervals = lm.selectStmt.Limit } } // If we are exceeding our MaxGroupByPoints error out if lm.numIntervals > MaxGroupByPoints { return errors.New("too many points in the group by interval. maybe you forgot to specify a where time clause?") } // Ensure that the start time for the results is on the start of the window. lm.queryTMinWindow = lm.queryTMin if lm.intervalSize > 0 { lm.queryTMinWindow = lm.queryTMinWindow / lm.intervalSize * lm.intervalSize } } selectFields := newStringSet() selectTags := newStringSet() whereFields := newStringSet() // Create the TagSet cursors for the Mapper. for _, src := range lm.selectStmt.Sources { mm, ok := src.(*influxql.Measurement) if !ok { return fmt.Errorf("invalid source type: %#v", src) } m := lm.shard.index.Measurement(mm.Name) if m == nil { // This shard have never received data for the measurement. No Mapper // required. return nil } // Validate that ANY GROUP BY is not a field for thie measurement. if err := m.ValidateGroupBy(lm.selectStmt); err != nil { return err } // Create tagset cursors and determine various field types within SELECT statement. tsf, err := createTagSetsAndFields(m, lm.selectStmt) if err != nil { return err } tagSets := tsf.tagSets selectFields.add(tsf.selectFields...) selectTags.add(tsf.selectTags...) whereFields.add(tsf.whereFields...) // Validate that any GROUP BY is not on a field if err := m.ValidateGroupBy(lm.selectStmt); err != nil { return err } // SLIMIT and SOFFSET the unique series if lm.selectStmt.SLimit > 0 || lm.selectStmt.SOffset > 0 { if lm.selectStmt.SOffset > len(tagSets) { tagSets = nil } else { if lm.selectStmt.SOffset+lm.selectStmt.SLimit > len(tagSets) { lm.selectStmt.SLimit = len(tagSets) - lm.selectStmt.SOffset } tagSets = tagSets[lm.selectStmt.SOffset : lm.selectStmt.SOffset+lm.selectStmt.SLimit] } } // Create all cursors for reading the data from this shard. for _, t := range tagSets { cursors := []*seriesCursor{} for i, key := range t.SeriesKeys { c := lm.tx.Cursor(key) if c == nil { // No data exists for this key. continue } cm := newSeriesCursor(c, t.Filters[i]) cursors = append(cursors, cm) } tsc := newTagSetCursor(m.Name, t.Tags, cursors, lm.shard.FieldCodec(m.Name)) // Prime the buffers. for i := 0; i < len(tsc.cursors); i++ { k, v := tsc.cursors[i].SeekTo(lm.queryTMin) tsc.keyBuffer[i] = k tsc.valueBuffer[i] = v } lm.cursors = append(lm.cursors, tsc) } sort.Sort(tagSetCursors(lm.cursors)) } lm.selectFields = selectFields.list() lm.selectTags = selectTags.list() lm.whereFields = whereFields.list() // If the query does not aggregate, then at least 1 SELECT field should be present. if lm.rawMode && len(lm.selectFields) == 0 { // None of the SELECT fields exist in this data. Wipe out all tagset cursors. lm.cursors = nil } return nil }