// mustParseSelectStatement parses a select statement. Panic on error.
func mustParseSelectStatement(s string) *influxql.SelectStatement {
stmt, err := influxql.NewParser(strings.NewReader(s)).ParseStatement()
if err != nil {
panic(err)
}
return stmt.(*influxql.SelectStatement)
}
// MustParseQuery parses an InfluxQL query. Panic on error.
func mustParseQuery(s string) *influxql.Query {
q, err := influxql.NewParser(strings.NewReader(s)).ParseQuery()
if err != nil {
panic(err.Error())
}
return q
}
// Ensure that the String() value of a statement is parseable
func TestParseString(t *testing.T) {
var tests = []struct {
stmt string
}{
{
stmt: `SELECT "cpu load" FROM myseries`,
},
{
stmt: `SELECT "cpu load" FROM "my series"`,
},
{
stmt: `SELECT "cpu\"load" FROM myseries`,
},
{
stmt: `SELECT "cpu'load" FROM myseries`,
},
{
stmt: `SELECT "cpu load" FROM "my\"series"`,
},
{
stmt: `SELECT * FROM myseries`,
},
}
for _, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
_, err = influxql.NewParser(strings.NewReader(stmt.String())).ParseStatement()
if err != nil {
t.Fatalf("failed to parse string: %v\norig: %v\ngot: %v", err, tt.stmt, stmt.String())
}
}
}
// Ensure the SELECT statement can extract GROUP BY interval.
func TestSelectStatement_GroupByInterval(t *testing.T) {
q := "SELECT sum(value) from foo where time < now() GROUP BY time(10m)"
stmt, err := influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s := stmt.(*influxql.SelectStatement)
d, err := s.GroupByInterval()
if d != 10*time.Minute {
t.Fatalf("group by interval not equal:\nexp=%s\ngot=%s", 10*time.Minute, d)
}
if err != nil {
t.Fatalf("error parsing group by interval: %s", err.Error())
}
}
// NewContinuousQuery returns a ContinuousQuery object with a parsed influxql.CreateContinuousQueryStatement
func NewContinuousQuery(database string, cqi *meta.ContinuousQueryInfo) (*ContinuousQuery, error) {
stmt, err := influxql.NewParser(strings.NewReader(cqi.Query)).ParseStatement()
if err != nil {
return nil, err
}
q, ok := stmt.(*influxql.CreateContinuousQueryStatement)
if !ok || q.Source.Target == nil || q.Source.Target.Measurement == nil {
return nil, errors.New("query isn't a valid continuous query")
}
cquery := &ContinuousQuery{
Database: database,
Info: cqi,
q: q.Source,
}
return cquery, nil
}
func (s *Store) CreateMapper(shardID uint64, query string, chunkSize int) (Mapper, error) {
q, err := influxql.NewParser(strings.NewReader(query)).ParseStatement()
if err != nil {
return nil, err
}
stmt, ok := q.(*influxql.SelectStatement)
if !ok {
return nil, fmt.Errorf("query is not a SELECT statement: %s", err.Error())
}
shard := s.Shard(shardID)
if shard == nil {
// This can happen if the shard has been assigned, but hasn't actually been created yet.
return nil, nil
}
if (stmt.IsRawQuery && !stmt.HasDistinct()) || stmt.IsSimpleDerivative() {
return NewRawMapper(shard, stmt, chunkSize), nil
}
return NewAggMapper(shard, stmt), nil
}
// Ensure that we see if a where clause has only time limitations
func TestSelectStatement_OnlyTimeDimensions(t *testing.T) {
var tests = []struct {
stmt string
exp bool
}{
{
stmt: `SELECT value FROM myseries WHERE value > 1`,
exp: false,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z'`,
exp: true,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z' AND time < '2000-01-01T00:00:05Z'`,
exp: true,
},
{
stmt: `SELECT value FROM foo WHERE time >= '2000-01-01T00:00:05Z' AND asdf = 'bar'`,
exp: false,
},
{
stmt: `SELECT value FROM foo WHERE asdf = 'jkl' AND (time >= '2000-01-01T00:00:05Z' AND time < '2000-01-01T00:00:05Z')`,
exp: false,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
if stmt.(*influxql.SelectStatement).OnlyTimeDimensions() != tt.exp {
t.Fatalf("%d. expected statement to return only time dimension to be %t: %s", i, tt.exp, tt.stmt)
}
}
}
// Test SELECT statement wildcard rewrite.
func TestSelectStatement_RewriteWildcards(t *testing.T) {
var fields = influxql.Fields{
&influxql.Field{Expr: &influxql.VarRef{Val: "value1"}},
&influxql.Field{Expr: &influxql.VarRef{Val: "value2"}},
}
var dimensions = influxql.Dimensions{
&influxql.Dimension{Expr: &influxql.VarRef{Val: "host"}},
&influxql.Dimension{Expr: &influxql.VarRef{Val: "region"}},
}
var tests = []struct {
stmt string
rewrite string
}{
// No wildcards
{
stmt: `SELECT value FROM cpu`,
rewrite: `SELECT value FROM cpu`,
},
// Query wildcard
{
stmt: `SELECT * FROM cpu`,
rewrite: `SELECT value1, value2 FROM cpu GROUP BY host, region`,
},
// Parser fundamentally prohibits multiple query sources
// Query wildcard with explicit
// {
// stmt: `SELECT *,value1 FROM cpu`,
// rewrite: `SELECT value1, value2, value1 FROM cpu`,
// },
// Query multiple wildcards
// {
// stmt: `SELECT *,* FROM cpu`,
// rewrite: `SELECT value1,value2,value1,value2 FROM cpu`,
// },
// No GROUP BY wildcards
{
stmt: `SELECT value FROM cpu GROUP BY host`,
rewrite: `SELECT value FROM cpu GROUP BY host`,
},
// No GROUP BY wildcards, time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY time(5ms)`,
},
// GROUP BY wildcard
{
stmt: `SELECT value FROM cpu GROUP BY *`,
rewrite: `SELECT value FROM cpu GROUP BY host, region`,
},
// GROUP BY wildcard with time
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m)`,
},
// GROUP BY wildarde with fill
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m) fill(0)`,
rewrite: `SELECT mean(value) FROM cpu WHERE time < now() GROUP BY host, region, time(1m) fill(0)`,
},
// GROUP BY wildcard with explicit
{
stmt: `SELECT value FROM cpu GROUP BY *,host`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host`,
},
// GROUP BY multiple wildcards
{
stmt: `SELECT value FROM cpu GROUP BY *,*`,
rewrite: `SELECT value FROM cpu GROUP BY host, region, host, region`,
},
// Combo
{
stmt: `SELECT * FROM cpu GROUP BY *`,
rewrite: `SELECT value1, value2 FROM cpu GROUP BY host, region`,
},
}
for i, tt := range tests {
t.Logf("index: %d, statement: %s", i, tt.stmt)
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Rewrite statement.
rw := stmt.(*influxql.SelectStatement).RewriteWildcards(fields, dimensions)
if rw == nil {
t.Errorf("%d. %q: unexpected nil statement", i, tt.stmt)
continue
}
if rw := rw.String(); tt.rewrite != rw {
t.Errorf("%d. %q: unexpected rewrite:\n\nexp=%s\n\ngot=%s\n\n", i, tt.stmt, tt.rewrite, rw)
continue
}
}
}
// Ensure the SELECT statement can extract substatements.
func TestSelectStatement_Substatement(t *testing.T) {
var tests = []struct {
stmt string
expr *influxql.VarRef
sub string
err string
}{
// 0. Single series
{
stmt: `SELECT value FROM myseries WHERE value > 1`,
expr: &influxql.VarRef{Val: "value"},
sub: `SELECT value FROM myseries WHERE value > 1.000`,
},
// 1. Simple join
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb`,
expr: &influxql.VarRef{Val: "aa.value"},
sub: `SELECT aa.value FROM aa`,
},
// 2. Simple merge
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT bb.value FROM bb`,
},
// 3. Join with condition
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE aa.host = 'servera' AND bb.host = 'serverb'`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT bb.value FROM bb WHERE bb.host = 'serverb'`,
},
// 4. Join with complex condition
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE aa.host = 'servera' AND (bb.host = 'serverb' OR bb.host = 'serverc') AND 1 = 2`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT bb.value FROM bb WHERE (bb.host = 'serverb' OR bb.host = 'serverc') AND 1.000 = 2.000`,
},
// 5. 4 with different condition order
{
stmt: `SELECT sum(aa.value) + sum(bb.value) FROM aa, bb WHERE ((bb.host = 'serverb' OR bb.host = 'serverc') AND aa.host = 'servera') AND 1 = 2`,
expr: &influxql.VarRef{Val: "bb.value"},
sub: `SELECT bb.value FROM bb WHERE ((bb.host = 'serverb' OR bb.host = 'serverc')) AND 1.000 = 2.000`,
},
}
for i, tt := range tests {
// Parse statement.
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Extract substatement.
sub, err := stmt.(*influxql.SelectStatement).Substatement(tt.expr)
if err != nil {
t.Errorf("%d. %q: unexpected error: %s", i, tt.stmt, err)
continue
}
if substr := sub.String(); tt.sub != substr {
t.Errorf("%d. %q: unexpected substatement:\n\nexp=%s\n\ngot=%s\n\n", i, tt.stmt, tt.sub, substr)
continue
}
}
}
func TestSelectStatement_HasWildcard(t *testing.T) {
var tests = []struct {
stmt string
wildcard bool
}{
// No wildcards
{
stmt: `SELECT value FROM cpu`,
wildcard: false,
},
// Query wildcard
{
stmt: `SELECT * FROM cpu`,
wildcard: true,
},
// No GROUP BY wildcards
{
stmt: `SELECT value FROM cpu GROUP BY host`,
wildcard: false,
},
// No GROUP BY wildcards, time only
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY time(5ms)`,
wildcard: false,
},
// GROUP BY wildcard
{
stmt: `SELECT value FROM cpu GROUP BY *`,
wildcard: true,
},
// GROUP BY wildcard with time
{
stmt: `SELECT mean(value) FROM cpu where time < now() GROUP BY *,time(1m)`,
wildcard: true,
},
// GROUP BY wildcard with explicit
{
stmt: `SELECT value FROM cpu GROUP BY *,host`,
wildcard: true,
},
// GROUP BY multiple wildcards
{
stmt: `SELECT value FROM cpu GROUP BY *,*`,
wildcard: true,
},
// Combo
{
stmt: `SELECT * FROM cpu GROUP BY *`,
wildcard: true,
},
}
for i, tt := range tests {
// Parse statement.
t.Logf("index: %d, statement: %s", i, tt.stmt)
stmt, err := influxql.NewParser(strings.NewReader(tt.stmt)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", tt.stmt, err)
}
// Test wildcard detection.
if w := stmt.(*influxql.SelectStatement).HasWildcard(); tt.wildcard != w {
t.Errorf("%d. %q: unexpected wildcard detection:\n\nexp=%v\n\ngot=%v\n\n", i, tt.stmt, tt.wildcard, w)
continue
}
}
}
// Ensure the SELECT statement can have its start and end time set
func TestSelectStatement_SetTimeRange(t *testing.T) {
q := "SELECT sum(value) from foo where time < now() GROUP BY time(10m)"
stmt, err := influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s := stmt.(*influxql.SelectStatement)
min, max := influxql.TimeRange(s.Condition)
start := time.Now().Add(-20 * time.Hour).Round(time.Second).UTC()
end := time.Now().Add(10 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max = influxql.TimeRange(s.Condition)
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure we can set a time on a select that already has one set
start = time.Now().Add(-20 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(10 * time.Hour).Round(time.Second).UTC()
q = fmt.Sprintf("SELECT sum(value) from foo WHERE time >= %ds and time <= %ds GROUP BY time(10m)", start.Unix(), end.Unix())
stmt, err = influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s = stmt.(*influxql.SelectStatement)
min, max = influxql.TimeRange(s.Condition)
if start != min || end != max {
t.Fatalf("start and end times weren't equal:\n exp: %s\n got: %s\n exp: %s\n got:%s\n", start, min, end, max)
}
// update and ensure it saves it
start = time.Now().Add(-40 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(20 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max = influxql.TimeRange(s.Condition)
// TODO: right now the SetTimeRange can't override the start time if it's more recent than what they're trying to set it to.
// shouldn't matter for our purposes with continuous queries, but fix this later
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure that when we set a time range other where clause conditions are still there
q = "SELECT sum(value) from foo WHERE foo = 'bar' and time < now() GROUP BY time(10m)"
stmt, err = influxql.NewParser(strings.NewReader(q)).ParseStatement()
if err != nil {
t.Fatalf("invalid statement: %q: %s", stmt, err)
}
s = stmt.(*influxql.SelectStatement)
// update and ensure it saves it
start = time.Now().Add(-40 * time.Hour).Round(time.Second).UTC()
end = time.Now().Add(20 * time.Hour).Round(time.Second).UTC()
s.SetTimeRange(start, end)
min, max = influxql.TimeRange(s.Condition)
if min != start {
t.Fatalf("start time wasn't set properly.\n exp: %s\n got: %s", start, min)
}
// the end range is actually one nanosecond before the given one since end is exclusive
end = end.Add(-time.Nanosecond)
if max != end {
t.Fatalf("end time wasn't set properly.\n exp: %s\n got: %s", end, max)
}
// ensure the where clause is there
hasWhere := false
influxql.WalkFunc(s.Condition, func(n influxql.Node) {
if ex, ok := n.(*influxql.BinaryExpr); ok {
if lhs, ok := ex.LHS.(*influxql.VarRef); ok {
if lhs.Val == "foo" {
if rhs, ok := ex.RHS.(*influxql.StringLiteral); ok {
if rhs.Val == "bar" {
hasWhere = true
}
}
}
}
}
})
if !hasWhere {
t.Fatal("set time range cleared out the where clause")
}
}
// serveQuery parses an incoming query and, if valid, executes the query.
func (h *Handler) serveQuery(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
q := r.URL.Query()
pretty := q.Get("pretty") == "true"
qp := strings.TrimSpace(q.Get("q"))
if qp == "" {
httpError(w, `missing required parameter "q"`, pretty, http.StatusBadRequest)
return
}
epoch := strings.TrimSpace(q.Get("epoch"))
p := influxql.NewParser(strings.NewReader(qp))
db := q.Get("db")
// Parse query from query string.
query, err := p.ParseQuery()
if err != nil {
httpError(w, "error parsing query: "+err.Error(), pretty, http.StatusBadRequest)
return
}
// Sanitize statements with passwords.
for _, s := range query.Statements {
switch stmt := s.(type) {
case *influxql.CreateUserStatement:
sanitize(r, stmt.Password)
case *influxql.SetPasswordUserStatement:
sanitize(r, stmt.Password)
}
}
// Check authorization.
if h.requireAuthentication {
err = h.QueryExecutor.Authorize(user, query, db)
if err != nil {
httpError(w, "error authorizing query: "+err.Error(), pretty, http.StatusUnauthorized)
return
}
}
// Parse chunk size. Use default if not provided or unparsable.
chunked := (q.Get("chunked") == "true")
chunkSize := DefaultChunkSize
if chunked {
if n, err := strconv.ParseInt(q.Get("chunk_size"), 10, 64); err == nil {
chunkSize = int(n)
}
}
// Execute query.
w.Header().Add("content-type", "application/json")
results, err := h.QueryExecutor.ExecuteQuery(query, db, chunkSize)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
// if we're not chunking, this will be the in memory buffer for all results before sending to client
resp := Response{Results: make([]*influxql.Result, 0)}
// Status header is OK once this point is reached.
w.WriteHeader(http.StatusOK)
// pull all results from the channel
for r := range results {
// Ignore nil results.
if r == nil {
continue
}
// if requested, convert result timestamps to epoch
if epoch != "" {
convertToEpoch(r, epoch)
}
// Write out result immediately if chunked.
if chunked {
w.Write(MarshalJSON(Response{
Results: []*influxql.Result{r},
}, pretty))
w.(http.Flusher).Flush()
continue
}
// It's not chunked so buffer results in memory.
// Results for statements need to be combined together.
// We need to check if this new result is for the same statement as
// the last result, or for the next statement
l := len(resp.Results)
if l == 0 {
resp.Results = append(resp.Results, r)
} else if resp.Results[l-1].StatementID == r.StatementID {
cr := resp.Results[l-1]
cr.Series = append(cr.Series, r.Series...)
} else {
resp.Results = append(resp.Results, r)
}
}
// If it's not chunked we buffered everything in memory, so write it out
if !chunked {
w.Write(MarshalJSON(resp, pretty))
}
}
