// Query InfluxDB and collect batches on batch collector. func (b *BatchNode) doQuery() error { defer b.ins[0].Close() if b.et.tm.InfluxDBService == nil { return errors.New("InfluxDB not configured, cannot query InfluxDB for batch query") } tickC := b.ticker.Start() for { select { case <-b.closing: return nil case <-b.aborting: return errors.New("batch doQuery aborted") case now := <-tickC: // Update times for query stop := now.Add(-1 * b.b.Offset) b.query.Start(stop.Add(-1 * b.b.Period)) b.query.Stop(stop) b.logger.Println("D! starting next batch query:", b.query.String()) // Connect con, err := b.et.tm.InfluxDBService.NewClient() if err != nil { return err } q := client.Query{ Command: b.query.String(), } // Execute query resp, err := con.Query(q) if err != nil { return err } if resp.Err != nil { return resp.Err } // Collect batches for _, res := range resp.Results { batches, err := models.ResultToBatches(res) if err != nil { return err } for _, bch := range batches { b.ins[0].CollectBatch(bch) } } } } }
// Record a series of batch queries defined by a batch task func (r *Service) doRecordBatch(rid uuid.UUID, t *kapacitor.Task, start, stop time.Time) error { et, err := kapacitor.NewExecutingTask(r.TaskMaster.New(), t) if err != nil { return err } batches, err := et.BatchQueries(start, stop) if err != nil { return err } if r.InfluxDBService == nil { return errors.New("InfluxDB not configured, cannot record batch query") } con, err := r.InfluxDBService.NewClient() if err != nil { return err } archive, err := r.newBatchArchive(rid) if err != nil { return err } for batchIdx, queries := range batches { w, err := archive.Create(batchIdx) if err != nil { return err } for _, q := range queries { query := client.Query{ Command: q, } resp, err := con.Query(query) if err != nil { return err } if resp.Err != nil { return resp.Err } for _, res := range resp.Results { batches, err := models.ResultToBatches(res) if err != nil { return err } for _, b := range batches { kapacitor.WriteBatchForRecording(w, b) } } } } return archive.Close() }
func (r *Service) runQueryBatch(source chan<- models.Batch, q string, cluster string) error { defer close(source) _, resp, err := r.execQuery(q, cluster) if err != nil { return err } // Write results to sources for _, res := range resp.Results { batches, err := models.ResultToBatches(res, false) if err != nil { return err } for _, batch := range batches { source <- batch } } return nil }
func (r *Service) doRecordQuery(rid uuid.UUID, q string, tt kapacitor.TaskType) error { // Parse query to determine dbrp var db, rp string s, err := influxql.ParseStatement(q) if err != nil { return err } if slct, ok := s.(*influxql.SelectStatement); ok && len(slct.Sources) == 1 { if m, ok := slct.Sources[0].(*influxql.Measurement); ok { db = m.Database rp = m.RetentionPolicy } } if db == "" || rp == "" { return errors.New("could not determine database and retention policy. Is the query fully qualified?") } // Query InfluxDB con, err := r.InfluxDBService.NewClient() if err != nil { return err } query := client.Query{ Command: q, } resp, err := con.Query(query) if err != nil { return err } if resp.Err != nil { return resp.Err } // Open appropriate writer var w io.Writer var c io.Closer switch tt { case kapacitor.StreamTask: sw, err := r.newStreamWriter(rid) if err != nil { return err } w = sw c = sw case kapacitor.BatchTask: archive, err := r.newBatchArchive(rid) if err != nil { return err } w, err = archive.Create(0) if err != nil { return err } c = archive } // Write results to writer for _, res := range resp.Results { batches, err := models.ResultToBatches(res) if err != nil { c.Close() return err } for _, batch := range batches { switch tt { case kapacitor.StreamTask: for _, bp := range batch.Points { p := models.Point{ Name: batch.Name, Database: db, RetentionPolicy: rp, Tags: bp.Tags, Fields: bp.Fields, Time: bp.Time, } kapacitor.WritePointForRecording(w, p, precision) } case kapacitor.BatchTask: kapacitor.WriteBatchForRecording(w, batch) } } } return c.Close() }
func (r *Service) runQueryStream(source chan<- models.Point, q, cluster string) error { defer close(source) dbrp, resp, err := r.execQuery(q, cluster) if err != nil { return err } // Write results to sources for _, res := range resp.Results { batches, err := models.ResultToBatches(res, false) if err != nil { return err } // Write points in order across batches // Find earliest time of first points current := time.Time{} for _, batch := range batches { if len(batch.Points) > 0 && (current.IsZero() || batch.Points[0].Time.Before(current)) { current = batch.Points[0].Time } } finishedCount := 0 batchCount := len(batches) for finishedCount != batchCount { next := time.Time{} for b := range batches { l := len(batches[b].Points) if l == 0 { finishedCount++ continue } i := 0 for ; i < l; i++ { bp := batches[b].Points[i] if bp.Time.After(current) { if next.IsZero() || bp.Time.Before(next) { next = bp.Time } break } // Write point p := models.Point{ Name: batches[b].Name, Database: dbrp.Database, RetentionPolicy: dbrp.RetentionPolicy, Tags: bp.Tags, Fields: bp.Fields, Time: bp.Time, } source <- p } // Remove written points batches[b].Points = batches[b].Points[i:] } current = next } } return nil }
func (s *Service) startRecordBatch(t *kapacitor.Task, start, stop time.Time) ([]<-chan models.Batch, <-chan error, error) { // We do not open the task master so it does not need to be closed et, err := kapacitor.NewExecutingTask(s.TaskMaster.New(""), t) if err != nil { return nil, nil, err } batches, err := et.BatchQueries(start, stop) if err != nil { return nil, nil, err } if s.InfluxDBService == nil { return nil, nil, errors.New("InfluxDB not configured, cannot record batch query") } sources := make([]<-chan models.Batch, len(batches)) errors := make(chan error, len(batches)) for batchIndex, batchQueries := range batches { source := make(chan models.Batch) sources[batchIndex] = source go func(cluster string, queries []*kapacitor.Query, groupByName bool) { defer close(source) // Connect to the cluster cli, err := s.InfluxDBService.NewNamedClient(cluster) if err != nil { errors <- err return } // Run queries for _, q := range queries { query := influxdb.Query{ Command: q.String(), } resp, err := cli.Query(query) if err != nil { errors <- err return } for _, res := range resp.Results { batches, err := models.ResultToBatches(res, groupByName) if err != nil { errors <- err return } for _, b := range batches { // Set stop time based off query bounds if b.TMax.IsZero() || !q.IsGroupedByTime() { b.TMax = q.StopTime() } source <- b } } } errors <- nil }(batchQueries.Cluster, batchQueries.Queries, batchQueries.GroupByMeasurement) } errC := make(chan error, 1) go func() { for i := 0; i < cap(errors); i++ { err := <-errors if err != nil { errC <- err return } } errC <- nil }() return sources, errC, nil }
// Query InfluxDB and collect batches on batch collector. func (b *QueryNode) doQuery() error { defer b.ins[0].Close() b.queryErrors = &expvar.Int{} b.batchesQueried = &expvar.Int{} b.pointsQueried = &expvar.Int{} b.statMap.Set(statsQueryErrors, b.queryErrors) b.statMap.Set(statsBatchesQueried, b.batchesQueried) b.statMap.Set(statsPointsQueried, b.pointsQueried) if b.et.tm.InfluxDBService == nil { return errors.New("InfluxDB not configured, cannot query InfluxDB for batch query") } con, err := b.et.tm.InfluxDBService.NewNamedClient(b.b.Cluster) if err != nil { return errors.Wrap(err, "failed to get InfluxDB client") } tickC := b.ticker.Start() for { select { case <-b.closing: return nil case <-b.aborting: return errors.New("batch doQuery aborted") case now := <-tickC: b.timer.Start() // Update times for query stop := now.Add(-1 * b.b.Offset) b.query.SetStartTime(stop.Add(-1 * b.b.Period)) b.query.SetStopTime(stop) qStr := b.query.String() b.logger.Println("D! starting next batch query:", qStr) // Execute query q := influxdb.Query{ Command: qStr, } resp, err := con.Query(q) if err != nil { b.queryErrors.Add(1) b.logger.Println("E!", err) b.timer.Stop() break } // Collect batches for _, res := range resp.Results { batches, err := models.ResultToBatches(res, b.byName) if err != nil { b.logger.Println("E! failed to understand query result:", err) b.queryErrors.Add(1) continue } for _, bch := range batches { // Set stop time based off query bounds if bch.TMax.IsZero() || !b.query.IsGroupedByTime() { bch.TMax = stop } b.batchesQueried.Add(1) b.pointsQueried.Add(int64(len(bch.Points))) b.timer.Pause() err := b.ins[0].CollectBatch(bch) if err != nil { return err } b.timer.Resume() } } b.timer.Stop() } } }
// Query InfluxDB and collect batches on batch collector. func (b *QueryNode) doQuery() error { defer b.ins[0].Close() b.queryErrors = &expvar.Int{} b.connectErrors = &expvar.Int{} b.batchesQueried = &expvar.Int{} b.pointsQueried = &expvar.Int{} b.statMap.Set(statsQueryErrors, b.queryErrors) b.statMap.Set(statsConnectErrors, b.connectErrors) b.statMap.Set(statsBatchesQueried, b.batchesQueried) b.statMap.Set(statsPointsQueried, b.pointsQueried) if b.et.tm.InfluxDBService == nil { return errors.New("InfluxDB not configured, cannot query InfluxDB for batch query") } var con influxdb.Client tickC := b.ticker.Start() for { select { case <-b.closing: return nil case <-b.aborting: return errors.New("batch doQuery aborted") case now := <-tickC: b.timer.Start() // Update times for query stop := now.Add(-1 * b.b.Offset) b.query.Start(stop.Add(-1 * b.b.Period)) b.query.Stop(stop) b.logger.Println("D! starting next batch query:", b.query.String()) var err error if con == nil { if b.b.Cluster != "" { con, err = b.et.tm.InfluxDBService.NewNamedClient(b.b.Cluster) } else { con, err = b.et.tm.InfluxDBService.NewDefaultClient() } if err != nil { b.logger.Println("E! failed to connect to InfluxDB:", err) b.connectErrors.Add(1) // Ensure connection is nil con = nil b.timer.Stop() break } } q := influxdb.Query{ Command: b.query.String(), } // Execute query resp, err := con.Query(q) if err != nil { b.logger.Println("E! query failed:", err) b.queryErrors.Add(1) // Get a new connection con = nil b.timer.Stop() break } if err := resp.Error(); err != nil { b.logger.Println("E! query returned error response:", err) b.queryErrors.Add(1) b.timer.Stop() break } // Collect batches for _, res := range resp.Results { batches, err := models.ResultToBatches(res, b.byName) if err != nil { b.logger.Println("E! failed to understand query result:", err) b.queryErrors.Add(1) continue } for _, bch := range batches { bch.TMax = stop b.batchesQueried.Add(1) b.pointsQueried.Add(int64(len(bch.Points))) b.timer.Pause() err := b.ins[0].CollectBatch(bch) if err != nil { return err } b.timer.Resume() } } b.timer.Stop() } } }