func emptyTestServer() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Influxdb-Version", SERVER_VERSION)
switch r.URL.Path {
case "/query":
values := r.URL.Query()
parser := influxql.NewParser(bytes.NewBufferString(values.Get("q")))
q, err := parser.ParseQuery()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
stmt := q.Statements[0]
switch stmt.(type) {
case *influxql.ShowDatabasesStatement:
io.WriteString(w, `{"results":[{"series":[{"name":"databases","columns":["name"],"values":[["db"]]}]}]}`)
case *influxql.ShowDiagnosticsStatement:
io.WriteString(w, `{"results":[{}]}`)
}
case "/write":
w.WriteHeader(http.StatusOK)
}
}))
}
// MustParseQuery parses an InfluxQL query. Panic on error.
func MustParseQuery(s string) *influxql.Query {
q, err := influxql.NewParser(strings.NewReader(s)).ParseQuery()
if err != nil {
panic(err.Error())
}
return q
}
func BenchmarkMeasurement_SeriesIDForExp_NERegex(b *testing.B) {
m := tsdb.NewMeasurement("cpu")
for i := 0; i < 100000; i++ {
s := tsdb.NewSeries("cpu", models.Tags{models.Tag{
Key: []byte("host"),
Value: []byte(fmt.Sprintf("host%d", i))}})
s.ID = uint64(i)
m.AddSeries(s)
}
if exp, got := 100000, len(m.SeriesKeys()); exp != got {
b.Fatalf("series count mismatch: exp %v got %v", exp, got)
}
stmt, err := influxql.NewParser(strings.NewReader(`SELECT * FROM cpu WHERE host !~ /foo\d+/`)).ParseStatement()
if err != nil {
b.Fatalf("invalid statement: %s", err)
}
selectStmt := stmt.(*influxql.SelectStatement)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ids := m.IDsForExpr(selectStmt.Condition.(*influxql.BinaryExpr))
if exp, got := 100000, len(ids); exp != got {
b.Fatalf("series count mismatch: exp %v got %v", exp, got)
}
}
}
func BenchmarkQuery_String(b *testing.B) {
p := influxql.NewParser(strings.NewReader(`SELECT foo AS zoo, a AS b FROM bar WHERE value > 10 AND q = 'hello'`))
q, _ := p.ParseStatement()
for i := 0; i < b.N; i++ {
_ = q.String()
}
}
func TestSelectStatement_IsSimpleDerivative(t *testing.T) {
var tests = []struct {
stmt string
derivative bool
}{
// No derivatives
{
stmt: `SELECT value FROM cpu`,
derivative: false,
},
// Query derivative
{
stmt: `SELECT derivative(value) FROM cpu`,
derivative: true,
},
// Query derivative
{
stmt: `SELECT non_negative_derivative(value) FROM cpu`,
derivative: true,
},
// No GROUP BY time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
derivative: false,
},
// No GROUP BY derivatives, time only
{
stmt: `SELECT non_negative_derivative(mean(value)) FROM cpu where time < now() GROUP BY time(5ms)`,
derivative: false,
},
// Invalid derivative function name
{
stmt: `SELECT typoDerivative(value) FROM cpu where time < now()`,
derivative: false,
},
}
for i, tt := range tests {
// Parse statement.
t.Logf("index: %d, statement: %s", i, tt.stmt)
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Test derivative detection.
if d := stmt.(*influxql.SelectStatement).IsSimpleDerivative(); tt.derivative != d {
t.Errorf("%d. %q: unexpected derivative detection:\n\nexp=%v\n\ngot=%v\n\n", i, tt.stmt, tt.derivative, d)
continue
}
}
}
func TestSelectStatement_HasCountDistinct(t *testing.T) {
var tests = []struct {
stmt string
count bool
}{
// No counts
{
stmt: `SELECT value FROM cpu`,
count: false,
},
// Query count
{
stmt: `SELECT count(value) FROM cpu`,
count: false,
},
// No GROUP BY time only
{
stmt: `SELECT count(distinct(value)) FROM cpu where time < now() GROUP BY time(5ms)`,
count: true,
},
// Query count
{
stmt: `SELECT typoCount(value) FROM cpu`,
count: false,
},
// No GROUP BY time only
{
stmt: `SELECT typoCount(distinct(value)) FROM cpu where time < now() GROUP BY time(5ms)`,
count: false,
},
}
for i, tt := range tests {
// Parse statement.
t.Logf("index: %d, statement: %s", i, tt.stmt)
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Test count detection.
if c := stmt.(*influxql.SelectStatement).HasCountDistinct(); tt.count != c {
t.Errorf("%d. %q: unexpected count detection:\n\nexp=%v\n\ngot=%v\n\n", i, tt.stmt, tt.count, c)
continue
}
}
}
// Test SELECT statement time field rewrite.
func TestSelectStatement_RewriteTimeFields(t *testing.T) {
var tests = []struct {
s string
stmt influxql.Statement
}{
{
s: `SELECT time, field1 FROM cpu`,
stmt: &influxql.SelectStatement{
IsRawQuery: true,
Fields: []*influxql.Field{
{Expr: &influxql.VarRef{Val: "field1"}},
},
Sources: []influxql.Source{
&influxql.Measurement{Name: "cpu"},
},
},
},
{
s: `SELECT time AS timestamp, field1 FROM cpu`,
stmt: &influxql.SelectStatement{
IsRawQuery: true,
Fields: []*influxql.Field{
{Expr: &influxql.VarRef{Val: "field1"}},
},
Sources: []influxql.Source{
&influxql.Measurement{Name: "cpu"},
},
TimeAlias: "timestamp",
},
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.s)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.s, err)
}
// Rewrite statement.
stmt.(*influxql.SelectStatement).RewriteTimeFields()
if !reflect.DeepEqual(tt.stmt, stmt) {
t.Logf("\n# %s\nexp=%s\ngot=%s\n", tt.s, mustMarshalJSON(tt.stmt), mustMarshalJSON(stmt))
t.Logf("\nSQL exp=%s\nSQL got=%s\n", tt.stmt.String(), stmt.String())
t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.stmt, stmt)
}
}
}
// Ensure the SELECT statement can extract GROUP BY interval.
func TestSelectStatement_GroupByInterval(t *testing.T) {
q := "SELECT sum(value) from foo where time < now() GROUP BY time(10m)"
stmt, err := influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s := stmt.(*influxql.SelectStatement)
d, err := s.GroupByInterval()
if d != 10*time.Minute {
t.Fatalf("group by interval not equal:\nexp=%s\ngot=%s", 10*time.Minute, d)
}
if err != nil {
t.Fatalf("error parsing group by interval: %s", err.Error())
}
}
func emptyTestServer() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Influxdb-Version", SERVER_VERSION)
// Fake authorization entirely based on the username.
authorized := false
user, _, _ := r.BasicAuth()
switch user {
case "", "admin":
authorized = true
}
switch r.URL.Path {
case "/query":
values := r.URL.Query()
parser := influxql.NewParser(bytes.NewBufferString(values.Get("q")))
q, err := parser.ParseQuery()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
stmt := q.Statements[0]
switch stmt.(type) {
case *influxql.ShowDatabasesStatement:
if authorized {
io.WriteString(w, `{"results":[{"series":[{"name":"databases","columns":["name"],"values":[["db"]]}]}]}`)
} else {
w.WriteHeader(http.StatusUnauthorized)
io.WriteString(w, fmt.Sprintf(`{"error":"error authorizing query: %s not authorized to execute statement 'SHOW DATABASES', requires admin privilege"}`, user))
}
case *influxql.ShowDiagnosticsStatement:
io.WriteString(w, `{"results":[{}]}`)
}
case "/write":
w.WriteHeader(http.StatusOK)
}
}))
}
// Ensure that we see if a where clause has only time limitations
func TestOnlyTimeExpr(t *testing.T) {
var tests = []struct {
stmt string
exp bool
}{
{
stmt: `SELECT value FROM myseries WHERE value > 1`,
exp: false,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z'`,
exp: true,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z' AND time < '2000-01-01T00:00:05Z'`,
exp: true,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z' AND asdf = 'bar'`,
exp: false,
},
{
stmt: `SELECT value FROM foo WHERE asdf = 'jkl' AND (time >= '2000-01-01T00:00:05Z' AND time < '2000-01-01T00:00:05Z')`,
exp: false,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
if influxql.OnlyTimeExpr(stmt.(*influxql.SelectStatement).Condition) != tt.exp {
t.Fatalf("%d. expected statement to return only time dimension to be %t: %s", i, tt.exp, tt.stmt)
}
}
}
// NewContinuousQuery returns a ContinuousQuery object with a parsed influxql.CreateContinuousQueryStatement
func NewContinuousQuery(database string, cqi *meta.ContinuousQueryInfo) (*ContinuousQuery, error) {
stmt, err := influxql.NewParser(strings.NewReader(cqi.Query)).ParseStatement()
if err != nil {
return nil, err
}
q, ok := stmt.(*influxql.CreateContinuousQueryStatement)
if !ok || q.Source.Target == nil || q.Source.Target.Measurement == nil {
return nil, errors.New("query isn't a valid continuous query")
}
cquery := &ContinuousQuery{
Database: database,
Info: cqi,
Resample: ResampleOptions{
Every: q.ResampleEvery,
For: q.ResampleFor,
},
q: q.Source,
}
return cquery, nil
}
// Ensure that the String() value of a statement is parseable
func TestParseString(t *testing.T) {
var tests = []struct {
stmt string
}{
{
stmt: `SELECT "cpu load" FROM myseries`,
},
{
stmt: `SELECT "cpu load" FROM "my series"`,
},
{
stmt: `SELECT "cpu\"load" FROM myseries`,
},
{
stmt: `SELECT "cpu'load" FROM myseries`,
},
{
stmt: `SELECT "cpu load" FROM "my\"series"`,
},
{
stmt: `SELECT "field with spaces" FROM "\"ugly\" db"."\"ugly\" rp"."\"ugly\" measurement"`,
},
{
stmt: `SELECT * FROM myseries`,
},
{
stmt: `DROP DATABASE "!"`,
},
{
stmt: `DROP RETENTION POLICY "my rp" ON "a database"`,
},
{
stmt: `CREATE RETENTION POLICY "my rp" ON "a database" DURATION 1d REPLICATION 1`,
},
{
stmt: `ALTER RETENTION POLICY "my rp" ON "a database" DEFAULT`,
},
{
stmt: `SHOW RETENTION POLICIES ON "a database"`,
},
{
stmt: `SHOW TAG VALUES WITH KEY IN ("a long name", short)`,
},
{
stmt: `DROP CONTINUOUS QUERY "my query" ON "my database"`,
},
// See issues https://github.com/influxdata/influxdb/issues/1647
// and https://github.com/influxdata/influxdb/issues/4404
//{
// stmt: `DELETE FROM "my db"."my rp"."my measurement"`,
//},
{
stmt: `DROP SUBSCRIPTION "ugly \"subscription\" name" ON "\"my\" db"."\"my\" rp"`,
},
{
stmt: `CREATE SUBSCRIPTION "ugly \"subscription\" name" ON "\"my\" db"."\"my\" rp" DESTINATIONS ALL 'my host', 'my other host'`,
},
{
stmt: `SHOW MEASUREMENTS WITH MEASUREMENT =~ /foo/`,
},
{
stmt: `SHOW MEASUREMENTS WITH MEASUREMENT = "and/or"`,
},
{
stmt: `DROP USER "user with spaces"`,
},
{
stmt: `GRANT ALL PRIVILEGES ON "db with spaces" TO "user with spaces"`,
},
{
stmt: `GRANT ALL PRIVILEGES TO "user with spaces"`,
},
{
stmt: `SHOW GRANTS FOR "user with spaces"`,
},
{
stmt: `REVOKE ALL PRIVILEGES ON "db with spaces" FROM "user with spaces"`,
},
{
stmt: `REVOKE ALL PRIVILEGES FROM "user with spaces"`,
},
{
stmt: `CREATE DATABASE "db with spaces"`,
},
}
for _, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
stmtCopy, err := influxql.NewParser(strings.NewReader(stmt.String())).ParseStatement()
if err != nil {
t.Fatalf("failed to parse string: %v\norig: %v\ngot: %v", err, tt.stmt, stmt.String())
}
if !reflect.DeepEqual(stmt, stmtCopy) {
t.Fatalf("statement changed after stringifying and re-parsing:\noriginal : %v\nre-parsed: %v\n", tt.stmt, stmtCopy.String())
}
}
}
// Ensure the SELECT statement can have its start and end time set
func TestSelectStatement_SetTimeRange(t *testing.T) {
q := "SELECT sum(value) from foo where time < now() GROUP BY time(10m)"
stmt, err := influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s := stmt.(*influxql.SelectStatement)
start := time.Now().Add(-20 * time.Hour).Round(time.Second).UTC()
end := time.Now().Add(10 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max := MustTimeRange(s.Condition)
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure we can set a time on a select that already has one set
start = time.Now().Add(-20 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(10 * time.Hour).Round(time.Second).UTC()
q = fmt.Sprintf("SELECT sum(value) from foo WHERE time >= %ds and time <= %ds GROUP BY time(10m)", start.Unix(), end.Unix())
stmt, err = influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s = stmt.(*influxql.SelectStatement)
min, max = MustTimeRange(s.Condition)
if start != min || end != max {
t.Fatalf("start and end times weren't equal:\n exp: %s\n got: %s\n exp: %s\n got:%s\n", start, min, end, max)
}
// update and ensure it saves it
start = time.Now().Add(-40 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(20 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max = MustTimeRange(s.Condition)
// TODO: right now the SetTimeRange can't override the start time if it's more recent than what they're trying to set it to.
// shouldn't matter for our purposes with continuous queries, but fix this later
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure that when we set a time range other where clause conditions are still there
q = "SELECT sum(value) from foo WHERE foo = 'bar' and time < now() GROUP BY time(10m)"
stmt, err = influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s = stmt.(*influxql.SelectStatement)
// update and ensure it saves it
start = time.Now().Add(-40 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(20 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max = MustTimeRange(s.Condition)
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure the where clause is there
hasWhere := false
influxql.WalkFunc(s.Condition, func(n influxql.Node) {
if ex, ok := n.(*influxql.BinaryExpr); ok {
if lhs, ok := ex.LHS.(*influxql.VarRef); ok {
if lhs.Val == "foo" {
if rhs, ok := ex.RHS.(*influxql.StringLiteral); ok {
if rhs.Val == "bar" {
hasWhere = true
}
}
}
}
}
})
if !hasWhere {
t.Fatal("set time range cleared out the where clause")
}
}
// serveQuery parses an incoming query and, if valid, executes the query.
func (h *Handler) serveQuery(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
h.statMap.Add(statQueryRequest, 1)
defer func(start time.Time) {
h.statMap.Add(statQueryRequestDuration, time.Since(start).Nanoseconds())
}(time.Now())
q := r.URL.Query()
pretty := q.Get("pretty") == "true"
qp := strings.TrimSpace(q.Get("q"))
if qp == "" {
httpError(w, `missing required parameter "q"`, pretty, http.StatusBadRequest)
return
}
epoch := strings.TrimSpace(q.Get("epoch"))
p := influxql.NewParser(strings.NewReader(qp))
db := q.Get("db")
// Parse query from query string.
query, err := p.ParseQuery()
if err != nil {
httpError(w, "error parsing query: "+err.Error(), pretty, http.StatusBadRequest)
return
}
// Sanitize statements with passwords.
for _, s := range query.Statements {
switch stmt := s.(type) {
case *influxql.CreateUserStatement:
sanitize(r, stmt.Password)
case *influxql.SetPasswordUserStatement:
sanitize(r, stmt.Password)
}
}
// Check authorization.
if h.requireAuthentication {
if err := h.QueryAuthorizer.AuthorizeQuery(user, query, db); err != nil {
if err, ok := err.(meta.ErrAuthorize); ok {
h.Logger.Printf("unauthorized request | user: %q | query: %q | database %q\n", err.User, err.Query.String(), err.Database)
}
httpError(w, "error authorizing query: "+err.Error(), pretty, http.StatusUnauthorized)
return
}
}
// Parse chunk size. Use default if not provided or unparsable.
chunked := (q.Get("chunked") == "true")
chunkSize := DefaultChunkSize
if chunked {
if n, err := strconv.ParseInt(q.Get("chunk_size"), 10, 64); err == nil {
chunkSize = int(n)
}
}
// Make sure if the client disconnects we signal the query to abort
closing := make(chan struct{})
if notifier, ok := w.(http.CloseNotifier); ok {
notify := notifier.CloseNotify()
go func() {
<-notify
close(closing)
}()
} else {
defer close(closing)
}
// Execute query.
w.Header().Add("content-type", "application/json")
results := h.QueryExecutor.ExecuteQuery(query, db, chunkSize, closing)
// if we're not chunking, this will be the in memory buffer for all results before sending to client
resp := Response{Results: make([]*influxql.Result, 0)}
// Status header is OK once this point is reached.
w.WriteHeader(http.StatusOK)
// pull all results from the channel
for r := range results {
// Ignore nil results.
if r == nil {
continue
}
// if requested, convert result timestamps to epoch
if epoch != "" {
convertToEpoch(r, epoch)
}
// Write out result immediately if chunked.
if chunked {
n, _ := w.Write(MarshalJSON(Response{
Results: []*influxql.Result{r},
}, pretty))
h.statMap.Add(statQueryRequestBytesTransmitted, int64(n))
w.(http.Flusher).Flush()
continue
}
// It's not chunked so buffer results in memory.
// Results for statements need to be combined together.
// We need to check if this new result is for the same statement as
// the last result, or for the next statement
l := len(resp.Results)
if l == 0 {
resp.Results = append(resp.Results, r)
} else if resp.Results[l-1].StatementID == r.StatementID {
cr := resp.Results[l-1]
rowsMerged := 0
if len(cr.Series) > 0 {
lastSeries := cr.Series[len(cr.Series)-1]
for _, row := range r.Series {
if !lastSeries.SameSeries(row) {
// Next row is for a different series than last.
break
}
// Values are for the same series, so append them.
lastSeries.Values = append(lastSeries.Values, row.Values...)
rowsMerged++
}
}
// Append remaining rows as new rows.
r.Series = r.Series[rowsMerged:]
cr.Series = append(cr.Series, r.Series...)
} else {
resp.Results = append(resp.Results, r)
}
}
// If it's not chunked we buffered everything in memory, so write it out
if !chunked {
n, _ := w.Write(MarshalJSON(resp, pretty))
h.statMap.Add(statQueryRequestBytesTransmitted, int64(n))
}
}
// Test SELECT statement field rewrite.
func TestSelectStatement_RewriteFields(t *testing.T) {
var tests = []struct {
stmt string
rewrite string
}{
// No wildcards
{
stmt: `SELECT value FROM cpu`,
rewrite: `SELECT value FROM cpu`,
},
// Query wildcard
{
stmt: `SELECT * FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer FROM cpu`,
},
// Parser fundamentally prohibits multiple query sources
// Query wildcard with explicit
{
stmt: `SELECT *,value1 FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer, value1::float FROM cpu`,
},
// Query multiple wildcards
{
stmt: `SELECT *,* FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer, host::tag, region::tag, value1::float, value2::integer FROM cpu`,
},
// Query wildcards with group by
{
stmt: `SELECT * FROM cpu GROUP BY host`,
rewrite: `SELECT region::tag, value1::float, value2::integer FROM cpu GROUP BY host`,
},
// No GROUP BY wildcards
{
stmt: `SELECT value FROM cpu GROUP BY host`,
rewrite: `SELECT value FROM cpu GROUP BY host`,
},
// No GROUP BY wildcards, time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY time(5ms)`,
},
// GROUP BY wildcard
{
stmt: `SELECT value FROM cpu GROUP BY *`,
rewrite: `SELECT value FROM cpu GROUP BY host, region`,
},
// GROUP BY wildcard with time
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m)`,
},
// GROUP BY wildcard with fill
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m) fill(0)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m) fill(0)`,
},
// GROUP BY wildcard with explicit
{
stmt: `SELECT value FROM cpu GROUP BY *,host`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host`,
},
// GROUP BY multiple wildcards
{
stmt: `SELECT value FROM cpu GROUP BY *,*`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host, region`,
},
// Combo
{
stmt: `SELECT * FROM cpu GROUP BY *`,
rewrite: `SELECT value1::float, value2::integer FROM cpu GROUP BY host, region`,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
var ic IteratorCreator
ic.FieldDimensionsFn = func(sources influxql.Sources) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) {
fields = map[string]influxql.DataType{"value1": influxql.Float, "value2": influxql.Integer}
dimensions = map[string]struct{}{"host": struct{}{}, "region": struct{}{}}
return
}
// Rewrite statement.
rw, err := stmt.(*influxql.SelectStatement).RewriteFields(&ic)
if err != nil {
t.Errorf("%d. %q: error: %s", i, tt.stmt, err)
} else if rw == nil {
t.Errorf("%d. %q: unexpected nil statement", i, tt.stmt)
} else if rw := rw.String(); tt.rewrite != rw {
t.Errorf("%d. %q: unexpected rewrite:\n\nexp=%s\n\ngot=%s\n\n", i, tt.stmt, tt.rewrite, rw)
}
}
}
// Ensure the SELECT statement can extract substatements.
func TestSelectStatement_Substatement(t *testing.T) {
var tests = []struct {
stmt string
expr *influxql.VarRef
sub string
err string
}{
// 0. Single series
{
stmt: `SELECT value FROM myseries WHERE value > 1`,
expr: &influxql.VarRef{Val: "value"},
sub: `SELECT value FROM myseries WHERE value > 1`,
},
// 1. Simple join
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb`,
expr: &influxql.VarRef{Val: "aa.value"},
sub: `SELECT "aa.value" FROM aa`,
},
// 2. Simple merge
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT "bb.value" FROM bb`,
},
// 3. Join with condition
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE aa.host = 'servera' AND bb.host = 'serverb'`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT "bb.value" FROM bb WHERE "bb.host" = 'serverb'`,
},
// 4. Join with complex condition
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE aa.host = 'servera' AND (bb.host = 'serverb' OR bb.host = 'serverc') AND 1 = 2`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT "bb.value" FROM bb WHERE ("bb.host" = 'serverb' OR "bb.host" = 'serverc') AND 1 = 2`,
},
// 5. 4 with different condition order
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE ((bb.host = 'serverb' OR bb.host = 'serverc') AND aa.host = 'servera') AND 1 = 2`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT "bb.value" FROM bb WHERE (("bb.host" = 'serverb' OR "bb.host" = 'serverc')) AND 1 = 2`,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Extract substatement.
sub, err := stmt.(*influxql.SelectStatement).Substatement(tt.expr)
if err != nil {
t.Errorf("%d. %q: unexpected error: %s", i, tt.stmt, err)
continue
}
if substr := sub.String(); tt.sub != substr {
t.Errorf("%d. %q: unexpected substatement:\n\nexp=%s\n\ngot=%s\n\n", i, tt.stmt, tt.sub, substr)
continue
}
}
}
// Test SELECT statement field rewrite.
func TestSelectStatement_RewriteFields(t *testing.T) {
var tests = []struct {
stmt string
rewrite string
}{
// No wildcards
{
stmt: `SELECT value FROM cpu`,
rewrite: `SELECT value FROM cpu`,
},
// Query wildcard
{
stmt: `SELECT * FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer FROM cpu`,
},
// Parser fundamentally prohibits multiple query sources
// Query wildcard with explicit
{
stmt: `SELECT *,value1 FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer, value1::float FROM cpu`,
},
// Query multiple wildcards
{
stmt: `SELECT *,* FROM cpu`,
rewrite: `SELECT host::tag, region::tag, value1::float, value2::integer, host::tag, region::tag, value1::float, value2::integer FROM cpu`,
},
// Query wildcards with group by
{
stmt: `SELECT * FROM cpu GROUP BY host`,
rewrite: `SELECT region::tag, value1::float, value2::integer FROM cpu GROUP BY host`,
},
// No GROUP BY wildcards
{
stmt: `SELECT value FROM cpu GROUP BY host`,
rewrite: `SELECT value FROM cpu GROUP BY host`,
},
// No GROUP BY wildcards, time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY time(5ms)`,
},
// GROUP BY wildcard
{
stmt: `SELECT value FROM cpu GROUP BY *`,
rewrite: `SELECT value FROM cpu GROUP BY host, region`,
},
// GROUP BY wildcard with time
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m)`,
},
// GROUP BY wildcard with fill
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m) fill(0)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m) fill(0)`,
},
// GROUP BY wildcard with explicit
{
stmt: `SELECT value FROM cpu GROUP BY *,host`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host`,
},
// GROUP BY multiple wildcards
{
stmt: `SELECT value FROM cpu GROUP BY *,*`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host, region`,
},
// Combo
{
stmt: `SELECT * FROM cpu GROUP BY *`,
rewrite: `SELECT value1::float, value2::integer FROM cpu GROUP BY host, region`,
},
// Wildcard function with all fields.
{
stmt: `SELECT mean(*) FROM cpu`,
rewrite: `SELECT mean(value1::float) AS mean_value1, mean(value2::integer) AS mean_value2 FROM cpu`,
},
{
stmt: `SELECT distinct(*) FROM strings`,
rewrite: `SELECT distinct(string::string) AS distinct_string, distinct(value::float) AS distinct_value FROM strings`,
},
{
stmt: `SELECT distinct(*) FROM bools`,
rewrite: `SELECT distinct(bool::boolean) AS distinct_bool, distinct(value::float) AS distinct_value FROM bools`,
},
// Wildcard function with some fields excluded.
{
stmt: `SELECT mean(*) FROM strings`,
rewrite: `SELECT mean(value::float) AS mean_value FROM strings`,
},
{
stmt: `SELECT mean(*) FROM bools`,
rewrite: `SELECT mean(value::float) AS mean_value FROM bools`,
},
// Wildcard function with an alias.
{
stmt: `SELECT mean(*) AS alias FROM cpu`,
rewrite: `SELECT mean(value1::float) AS alias_value1, mean(value2::integer) AS alias_value2 FROM cpu`,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
var ic IteratorCreator
ic.FieldDimensionsFn = func(sources influxql.Sources) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) {
source := sources[0].(*influxql.Measurement)
switch source.Name {
case "cpu":
fields = map[string]influxql.DataType{
"value1": influxql.Float,
"value2": influxql.Integer,
}
case "strings":
fields = map[string]influxql.DataType{
"value": influxql.Float,
"string": influxql.String,
}
case "bools":
fields = map[string]influxql.DataType{
"value": influxql.Float,
"bool": influxql.Boolean,
}
}
dimensions = map[string]struct{}{"host": struct{}{}, "region": struct{}{}}
return
}
// Rewrite statement.
rw, err := stmt.(*influxql.SelectStatement).RewriteFields(&ic)
if err != nil {
t.Errorf("%d. %q: error: %s", i, tt.stmt, err)
} else if rw == nil {
t.Errorf("%d. %q: unexpected nil statement", i, tt.stmt)
} else if rw := rw.String(); tt.rewrite != rw {
t.Errorf("%d. %q: unexpected rewrite:\n\nexp=%s\n\ngot=%s\n\n", i, tt.stmt, tt.rewrite, rw)
}
}
}
func TestSelectStatement_HasWildcard(t *testing.T) {
var tests = []struct {
stmt string
wildcard bool
}{
// No wildcards
{
stmt: `SELECT value FROM cpu`,
wildcard: false,
},
// Query wildcard
{
stmt: `SELECT * FROM cpu`,
wildcard: true,
},
// No GROUP BY wildcards
{
stmt: `SELECT value FROM cpu GROUP BY host`,
wildcard: false,
},
// No GROUP BY wildcards, time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
wildcard: false,
},
// GROUP BY wildcard
{
stmt: `SELECT value FROM cpu GROUP BY *`,
wildcard: true,
},
// GROUP BY wildcard with time
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m)`,
wildcard: true,
},
// GROUP BY wildcard with explicit
{
stmt: `SELECT value FROM cpu GROUP BY *,host`,
wildcard: true,
},
// GROUP BY multiple wildcards
{
stmt: `SELECT value FROM cpu GROUP BY *,*`,
wildcard: true,
},
// Combo
{
stmt: `SELECT * FROM cpu GROUP BY *`,
wildcard: true,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Test wildcard detection.
if w := stmt.(*influxql.SelectStatement).HasWildcard(); tt.wildcard != w {
t.Errorf("%d. %q: unexpected wildcard detection:\n\nexp=%v\n\ngot=%v\n\n", i, tt.stmt, tt.wildcard, w)
continue
}
}
}
// serveQuery parses an incoming query and, if valid, executes the query.
func (h *Handler) serveQuery(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
atomic.AddInt64(&h.stats.QueryRequests, 1)
defer func(start time.Time) {
atomic.AddInt64(&h.stats.QueryRequestDuration, time.Since(start).Nanoseconds())
}(time.Now())
pretty := r.FormValue("pretty") == "true"
nodeID, _ := strconv.ParseUint(r.FormValue("node_id"), 10, 64)
qp := strings.TrimSpace(r.FormValue("q"))
if qp == "" {
h.httpError(w, `missing required parameter "q"`, pretty, http.StatusBadRequest)
return
}
epoch := strings.TrimSpace(r.FormValue("epoch"))
p := influxql.NewParser(strings.NewReader(qp))
db := r.FormValue("db")
// Sanitize the request query params so it doesn't show up in the response logger.
// Do this before anything else so a parsing error doesn't leak passwords.
sanitize(r)
// Parse the parameters
rawParams := r.FormValue("params")
if rawParams != "" {
var params map[string]interface{}
decoder := json.NewDecoder(strings.NewReader(rawParams))
decoder.UseNumber()
if err := decoder.Decode(¶ms); err != nil {
h.httpError(w, "error parsing query parameters: "+err.Error(), pretty, http.StatusBadRequest)
return
}
// Convert json.Number into int64 and float64 values
for k, v := range params {
if v, ok := v.(json.Number); ok {
var err error
if strings.Contains(string(v), ".") {
params[k], err = v.Float64()
} else {
params[k], err = v.Int64()
}
if err != nil {
h.httpError(w, "error parsing json value: "+err.Error(), pretty, http.StatusBadRequest)
return
}
}
}
p.SetParams(params)
}
// Parse query from query string.
query, err := p.ParseQuery()
if err != nil {
h.httpError(w, "error parsing query: "+err.Error(), pretty, http.StatusBadRequest)
return
}
// Check authorization.
if h.Config.AuthEnabled {
if err := h.QueryAuthorizer.AuthorizeQuery(user, query, db); err != nil {
if err, ok := err.(meta.ErrAuthorize); ok {
h.Logger.Printf("Unauthorized request | user: %q | query: %q | database %q\n", err.User, err.Query.String(), err.Database)
}
h.httpError(w, "error authorizing query: "+err.Error(), pretty, http.StatusUnauthorized)
return
}
}
// Parse chunk size. Use default if not provided or unparsable.
chunked := (r.FormValue("chunked") == "true")
chunkSize := DefaultChunkSize
if chunked {
if n, err := strconv.ParseInt(r.FormValue("chunk_size"), 10, 64); err == nil && int(n) > 0 {
chunkSize = int(n)
}
}
// Make sure if the client disconnects we signal the query to abort
closing := make(chan struct{})
if notifier, ok := w.(http.CloseNotifier); ok {
// CloseNotify() is not guaranteed to send a notification when the query
// is closed. Use this channel to signal that the query is finished to
// prevent lingering goroutines that may be stuck.
done := make(chan struct{})
defer close(done)
notify := notifier.CloseNotify()
go func() {
// Wait for either the request to finish
// or for the client to disconnect
select {
case <-done:
case <-notify:
close(closing)
}
}()
} else {
defer close(closing)
}
// Execute query.
w.Header().Add("Connection", "close")
w.Header().Add("Content-Type", "application/json")
results := h.QueryExecutor.ExecuteQuery(query, influxql.ExecutionOptions{
Database: db,
ChunkSize: chunkSize,
ReadOnly: r.Method == "GET",
NodeID: nodeID,
}, closing)
// if we're not chunking, this will be the in memory buffer for all results before sending to client
resp := Response{Results: make([]*influxql.Result, 0)}
// Status header is OK once this point is reached.
h.writeHeader(w, http.StatusOK)
// pull all results from the channel
rows := 0
for r := range results {
// Ignore nil results.
if r == nil {
continue
}
// if requested, convert result timestamps to epoch
if epoch != "" {
convertToEpoch(r, epoch)
}
// Write out result immediately if chunked.
if chunked {
n, _ := w.Write(MarshalJSON(Response{
Results: []*influxql.Result{r},
}, pretty))
if !pretty {
w.Write([]byte("\n"))
}
atomic.AddInt64(&h.stats.QueryRequestBytesTransmitted, int64(n))
w.(http.Flusher).Flush()
continue
}
// Limit the number of rows that can be returned in a non-chunked response.
// This is to prevent the server from going OOM when returning a large response.
// If you want to return more than the default chunk size, then use chunking
// to process multiple blobs.
rows += len(r.Series)
if h.Config.MaxRowLimit > 0 && rows > h.Config.MaxRowLimit {
break
}
// It's not chunked so buffer results in memory.
// Results for statements need to be combined together.
// We need to check if this new result is for the same statement as
// the last result, or for the next statement
l := len(resp.Results)
if l == 0 {
resp.Results = append(resp.Results, r)
} else if resp.Results[l-1].StatementID == r.StatementID {
if r.Err != nil {
resp.Results[l-1] = r
continue
}
cr := resp.Results[l-1]
rowsMerged := 0
if len(cr.Series) > 0 {
lastSeries := cr.Series[len(cr.Series)-1]
for _, row := range r.Series {
if !lastSeries.SameSeries(row) {
// Next row is for a different series than last.
break
}
// Values are for the same series, so append them.
lastSeries.Values = append(lastSeries.Values, row.Values...)
rowsMerged++
}
}
// Append remaining rows as new rows.
r.Series = r.Series[rowsMerged:]
cr.Series = append(cr.Series, r.Series...)
cr.Messages = append(cr.Messages, r.Messages...)
} else {
resp.Results = append(resp.Results, r)
}
}
// If it's not chunked we buffered everything in memory, so write it out
if !chunked {
n, _ := w.Write(MarshalJSON(resp, pretty))
atomic.AddInt64(&h.stats.QueryRequestBytesTransmitted, int64(n))
}
}
// serveQuery parses an incoming query and, if valid, executes the query.
func (h *Handler) serveQuery(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
atomic.AddInt64(&h.stats.QueryRequests, 1)
defer func(start time.Time) {
atomic.AddInt64(&h.stats.QueryRequestDuration, time.Since(start).Nanoseconds())
}(time.Now())
// Retrieve the underlying ResponseWriter or initialize our own.
rw, ok := w.(ResponseWriter)
if !ok {
rw = NewResponseWriter(w, r)
}
// Retrieve the node id the query should be executed on.
nodeID, _ := strconv.ParseUint(r.FormValue("node_id"), 10, 64)
var qr io.Reader
// Attempt to read the form value from the "q" form value.
if qp := strings.TrimSpace(r.FormValue("q")); qp != "" {
qr = strings.NewReader(qp)
} else if r.MultipartForm != nil && r.MultipartForm.File != nil {
// If we have a multipart/form-data, try to retrieve a file from 'q'.
if fhs := r.MultipartForm.File["q"]; len(fhs) > 0 {
f, err := fhs[0].Open()
if err != nil {
h.httpError(rw, err.Error(), http.StatusBadRequest)
return
}
defer f.Close()
qr = f
}
}
if qr == nil {
h.httpError(rw, `missing required parameter "q"`, http.StatusBadRequest)
return
}
epoch := strings.TrimSpace(r.FormValue("epoch"))
p := influxql.NewParser(qr)
db := r.FormValue("db")
// Sanitize the request query params so it doesn't show up in the response logger.
// Do this before anything else so a parsing error doesn't leak passwords.
sanitize(r)
// Parse the parameters
rawParams := r.FormValue("params")
if rawParams != "" {
var params map[string]interface{}
decoder := json.NewDecoder(strings.NewReader(rawParams))
decoder.UseNumber()
if err := decoder.Decode(¶ms); err != nil {
h.httpError(rw, "error parsing query parameters: "+err.Error(), http.StatusBadRequest)
return
}
// Convert json.Number into int64 and float64 values
for k, v := range params {
if v, ok := v.(json.Number); ok {
var err error
if strings.Contains(string(v), ".") {
params[k], err = v.Float64()
} else {
params[k], err = v.Int64()
}
if err != nil {
h.httpError(rw, "error parsing json value: "+err.Error(), http.StatusBadRequest)
return
}
}
}
p.SetParams(params)
}
// Parse query from query string.
query, err := p.ParseQuery()
if err != nil {
h.httpError(rw, "error parsing query: "+err.Error(), http.StatusBadRequest)
return
}
// Check authorization.
if h.Config.AuthEnabled {
if err := h.QueryAuthorizer.AuthorizeQuery(user, query, db); err != nil {
if err, ok := err.(meta.ErrAuthorize); ok {
h.Logger.Printf("Unauthorized request | user: %q | query: %q | database %q\n", err.User, err.Query.String(), err.Database)
}
h.httpError(rw, "error authorizing query: "+err.Error(), http.StatusForbidden)
return
}
}
// Parse chunk size. Use default if not provided or unparsable.
chunked := r.FormValue("chunked") == "true"
chunkSize := DefaultChunkSize
if chunked {
if n, err := strconv.ParseInt(r.FormValue("chunk_size"), 10, 64); err == nil && int(n) > 0 {
chunkSize = int(n)
}
}
// Parse whether this is an async command.
async := r.FormValue("async") == "true"
opts := influxql.ExecutionOptions{
Database: db,
ChunkSize: chunkSize,
ReadOnly: r.Method == "GET",
NodeID: nodeID,
}
// Make sure if the client disconnects we signal the query to abort
var closing chan struct{}
if !async {
closing = make(chan struct{})
if notifier, ok := w.(http.CloseNotifier); ok {
// CloseNotify() is not guaranteed to send a notification when the query
// is closed. Use this channel to signal that the query is finished to
// prevent lingering goroutines that may be stuck.
done := make(chan struct{})
defer close(done)
notify := notifier.CloseNotify()
go func() {
// Wait for either the request to finish
// or for the client to disconnect
select {
case <-done:
case <-notify:
close(closing)
}
}()
opts.AbortCh = done
} else {
defer close(closing)
}
}
// Execute query.
rw.Header().Add("Connection", "close")
results := h.QueryExecutor.ExecuteQuery(query, opts, closing)
// If we are running in async mode, open a goroutine to drain the results
// and return with a StatusNoContent.
if async {
go h.async(query, results)
h.writeHeader(w, http.StatusNoContent)
return
}
// if we're not chunking, this will be the in memory buffer for all results before sending to client
resp := Response{Results: make([]*influxql.Result, 0)}
// Status header is OK once this point is reached.
// Attempt to flush the header immediately so the client gets the header information
// and knows the query was accepted.
h.writeHeader(rw, http.StatusOK)
if w, ok := w.(http.Flusher); ok {
w.Flush()
}
// pull all results from the channel
rows := 0
for r := range results {
// Ignore nil results.
if r == nil {
continue
}
// if requested, convert result timestamps to epoch
if epoch != "" {
convertToEpoch(r, epoch)
}
// Write out result immediately if chunked.
if chunked {
n, _ := rw.WriteResponse(Response{
Results: []*influxql.Result{r},
})
atomic.AddInt64(&h.stats.QueryRequestBytesTransmitted, int64(n))
w.(http.Flusher).Flush()
continue
}
// Limit the number of rows that can be returned in a non-chunked
// response. This is to prevent the server from going OOM when
// returning a large response. If you want to return more than the
// default chunk size, then use chunking to process multiple blobs.
// Iterate through the series in this result to count the rows and
// truncate any rows we shouldn't return.
if h.Config.MaxRowLimit > 0 {
for i, series := range r.Series {
n := h.Config.MaxRowLimit - rows
if n < len(series.Values) {
// We have reached the maximum number of values. Truncate
// the values within this row.
series.Values = series.Values[:n]
// Since this was truncated, it will always be a partial return.
// Add this so the client knows we truncated the response.
series.Partial = true
}
rows += len(series.Values)
if rows >= h.Config.MaxRowLimit {
// Drop any remaining series since we have already reached the row limit.
if i < len(r.Series) {
r.Series = r.Series[:i+1]
}
break
}
}
}
// It's not chunked so buffer results in memory.
// Results for statements need to be combined together.
// We need to check if this new result is for the same statement as
// the last result, or for the next statement
l := len(resp.Results)
if l == 0 {
resp.Results = append(resp.Results, r)
} else if resp.Results[l-1].StatementID == r.StatementID {
if r.Err != nil {
resp.Results[l-1] = r
continue
}
cr := resp.Results[l-1]
rowsMerged := 0
if len(cr.Series) > 0 {
lastSeries := cr.Series[len(cr.Series)-1]
for _, row := range r.Series {
if !lastSeries.SameSeries(row) {
// Next row is for a different series than last.
break
}
// Values are for the same series, so append them.
lastSeries.Values = append(lastSeries.Values, row.Values...)
rowsMerged++
}
}
// Append remaining rows as new rows.
r.Series = r.Series[rowsMerged:]
cr.Series = append(cr.Series, r.Series...)
cr.Messages = append(cr.Messages, r.Messages...)
cr.Partial = r.Partial
} else {
resp.Results = append(resp.Results, r)
}
// Drop out of this loop and do not process further results when we hit the row limit.
if h.Config.MaxRowLimit > 0 && rows >= h.Config.MaxRowLimit {
// If the result is marked as partial, remove that partial marking
// here. While the series is partial and we would normally have
// tried to return the rest in the next chunk, we are not using
// chunking and are truncating the series so we don't want to
// signal to the client that we plan on sending another JSON blob
// with another result. The series, on the other hand, still
// returns partial true if it was truncated or had more data to
// send in a future chunk.
r.Partial = false
break
}
}
// If it's not chunked we buffered everything in memory, so write it out
if !chunked {
n, _ := rw.WriteResponse(resp)
atomic.AddInt64(&h.stats.QueryRequestBytesTransmitted, int64(n))
}
}
// Test SELECT statement regex conditions rewrite.
func TestSelectStatement_RewriteRegexConditions(t *testing.T) {
var tests = []struct {
in string
out string
}{
{in: `SELECT value FROM cpu`, out: `SELECT value FROM cpu`},
{in: `SELECT value FROM cpu WHERE host='server-1'`, out: `SELECT value FROM cpu WHERE host='server-1'`},
{in: `SELECT value FROM cpu WHERE host = 'server-1'`, out: `SELECT value FROM cpu WHERE host = 'server-1'`},
{in: `SELECT value FROM cpu WHERE host != 'server-1'`, out: `SELECT value FROM cpu WHERE host != 'server-1'`},
// Non matching regex
{in: `SELECT value FROM cpu WHERE host =~ /server-1|server-2|server-3/`, out: `SELECT value FROM cpu WHERE host =~ /server-1|server-2|server-3/`},
{in: `SELECT value FROM cpu WHERE host =~ /server-1/`, out: `SELECT value FROM cpu WHERE host =~ /server-1/`},
{in: `SELECT value FROM cpu WHERE host !~ /server-1/`, out: `SELECT value FROM cpu WHERE host !~ /server-1/`},
{in: `SELECT value FROM cpu WHERE host =~ /^server-1/`, out: `SELECT value FROM cpu WHERE host =~ /^server-1/`},
{in: `SELECT value FROM cpu WHERE host =~ /server-1$/`, out: `SELECT value FROM cpu WHERE host =~ /server-1$/`},
{in: `SELECT value FROM cpu WHERE host !~ /\^server-1$/`, out: `SELECT value FROM cpu WHERE host !~ /\^server-1$/`},
{in: `SELECT value FROM cpu WHERE host !~ /\^$/`, out: `SELECT value FROM cpu WHERE host !~ /\^$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^server-1\$/`, out: `SELECT value FROM cpu WHERE host !~ /^server-1\$/`},
{in: `SELECT value FROM cpu WHERE host =~ /^\$/`, out: `SELECT value FROM cpu WHERE host =~ /^\$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^a/`, out: `SELECT value FROM cpu WHERE host !~ /^a/`},
// These regexes are not supported due to the presence of escaped or meta characters.
{in: `SELECT value FROM cpu WHERE host !~ /^(foo|bar)$/`, out: `SELECT value FROM cpu WHERE host !~ /^(foo|bar)$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^?a$/`, out: `SELECT value FROM cpu WHERE host !~ /^?a$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^[a-z]$/`, out: `SELECT value FROM cpu WHERE host !~ /^[a-z]$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^\d$/`, out: `SELECT value FROM cpu WHERE host !~ /^\d$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^a*$/`, out: `SELECT value FROM cpu WHERE host !~ /^a*$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^a.b$/`, out: `SELECT value FROM cpu WHERE host !~ /^a.b$/`},
{in: `SELECT value FROM cpu WHERE host !~ /^ab+$/`, out: `SELECT value FROM cpu WHERE host !~ /^ab+$/`},
{in: `SELECT value FROM cpu WHERE host =~ /^hello\world$/`, out: `SELECT value FROM cpu WHERE host =~ /^hello\world$/`},
// These regexes all match and will be rewritten.
{in: `SELECT value FROM cpu WHERE host !~ /^a[2]$/`, out: `SELECT value FROM cpu WHERE host != 'a2'`},
{in: `SELECT value FROM cpu WHERE host =~ /^server-1$/`, out: `SELECT value FROM cpu WHERE host = 'server-1'`},
{in: `SELECT value FROM cpu WHERE host !~ /^server-1$/`, out: `SELECT value FROM cpu WHERE host != 'server-1'`},
{in: `SELECT value FROM cpu WHERE host =~ /^server 1$/`, out: `SELECT value FROM cpu WHERE host = 'server 1'`},
{in: `SELECT value FROM cpu WHERE host =~ /^$/`, out: `SELECT value FROM cpu WHERE host = ''`},
{in: `SELECT value FROM cpu WHERE host !~ /^$/`, out: `SELECT value FROM cpu WHERE host != ''`},
{in: `SELECT value FROM cpu WHERE host =~ /^server-1$/ OR host =~ /^server-2$/`, out: `SELECT value FROM cpu WHERE host = 'server-1' OR host = 'server-2'`},
{in: `SELECT value FROM cpu WHERE host =~ /^server-1$/ OR host =~ /^server]a$/`, out: `SELECT value FROM cpu WHERE host = 'server-1' OR host = 'server]a'`},
{in: `SELECT value FROM cpu WHERE host =~ /^hello\?$/`, out: `SELECT value FROM cpu WHERE host = 'hello?'`},
{in: `SELECT value FROM cpu WHERE host !~ /^\\$/`, out: `SELECT value FROM cpu WHERE host != '\\'`},
{in: `SELECT value FROM cpu WHERE host !~ /^\\\$$/`, out: `SELECT value FROM cpu WHERE host != '\\$'`},
}
for i, test := range tests {
stmt, err := influxql.NewParser(strings.NewReader(test.in)).ParseStatement()
if err != nil {
t.Fatalf("[Example %d], %v", i, err)
}
// Rewrite any supported regex conditions.
stmt.(*influxql.SelectStatement).RewriteRegexConditions()
// Get the expected rewritten statement.
expStmt, err := influxql.NewParser(strings.NewReader(test.out)).ParseStatement()
if err != nil {
t.Fatalf("[Example %d], %v", i, err)
}
// Compare the (potentially) rewritten AST to the expected AST.
if got, exp := stmt, expStmt; !reflect.DeepEqual(got, exp) {
t.Errorf("[Example %d]\nattempting %v\ngot %v\n%s\n\nexpected %v\n%s\n", i+1, test.in, got, mustMarshalJSON(got), exp, mustMarshalJSON(exp))
}
}
}