func TestSqlParseFromTypes(t *testing.T) {
t.Parallel()
sql := `select gh.repository.name, gh.id, gp.date
FROM github_fork as gh
INNER JOIN github_push AS gp ON gp.repo_id = gh.repo_id
WHERE
gh.repository.language = "go"
`
req, err := ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok := req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
sel.Rewrite()
assert.Tf(t, len(sel.From) == 2, "wanted 2 froms but got %v", len(sel.From))
//assert.Tf(t, len(sel.OrderBy) == 1, "want 1 orderby but has %v", len(sel.OrderBy))
u.Info(sel.String())
// Try compound join keys
sql = `select u.fname, u.lname, u.userid, b.description
FROM user as u
INNER JOIN blog AS b ON b.first_name = u.fname AND b.last_name = u.lname
`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel = req.(*SqlSelect)
sel.Rewrite()
user := sel.From[0]
//u.Infof("join nodes: %q", user.JoinExpr.String())
//blog := sel.From[1]
assert.Tf(t, user.Source != nil, "")
// 3 join tables
sql = `
SELECT
t1.name, t2.salary, t3.price
FROM employee AS t1
INNER JOIN info AS t2
ON t1.name = t2.name
INNER JOIN orders AS t3
ON t3.id = t2.fake_id;`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, len(sel.From) == 3, "has 3 from: %v", sel.From)
//assert.Tf(t, len(sel.OrderBy) == 1, "want 1 orderby but has %v", len(sel.OrderBy))
u.Info(sel.String())
}
func main() {
baseDir, err := os.Getwd()
quitIfErr(err)
skipDirFlag := flag.String("skip", "trash", "Comma-separated list of directories to skip")
buildOnlyFlag := flag.Bool("buildOnly", false, "Do \"go build\" instead of \"go test\"")
shortFlag := flag.Bool("short", false, `Run "go test" with "short" flag`)
colorFlag := flag.Bool("c", true, "Use colorized log output, colored by severity")
verboseFlag := flag.Bool("v", false, `Run "go test" with -v, also be more verbose elsewhere`)
veryVerbose := flag.Bool("vv", false, `Very Verbose, comine stdout AND stderr to display`)
raceFlag := flag.Bool("race", false, `Run "go test" with "race" flag`)
flag.Parse()
gou.SetLogger(log.New(os.Stderr, "", 0), "debug")
if *colorFlag {
gou.SetColorIfTerminal()
}
skipDirNames := strings.Split(*skipDirFlag, ",")
skipDirStats := make([]os.FileInfo, 0)
for _, skipDirName := range skipDirNames {
if skipDirName == "" {
continue
}
stat, err := os.Stat(skipDirName)
if skipDirName == "trash" && err != nil {
continue
}
if err != nil {
gou.Errorf("Couldn't stat directory to skip %s: %s\n", skipDirName, err)
}
skipDirStats = append(skipDirStats, stat)
}
conf := NewConf(skipDirStats, *buildOnlyFlag, *shortFlag, *raceFlag, *verboseFlag, *veryVerbose)
failedDirs := RunTestsRecursively(baseDir, baseDir, conf)
if len(failedDirs) > 0 {
gou.Error("\nFailed directories:")
for _, dir := range failedDirs {
gou.Errorf(" %s", dir)
}
os.Exit(1)
} else {
gou.Info("\nall tests/builds succeeded")
}
}
// Flush all current documents to ElasticSearch
func (b *BulkIndexor) Flush() {
b.mu.Lock()
if b.docCt > 0 {
b.send(b.buf)
}
b.mu.Unlock()
for {
select {
case <-wgChan(b.sendWg):
// done
u.Info("Normal Wait Group Shutdown")
return
case <-time.After(time.Second * time.Duration(MAX_SHUTDOWN_SECS)):
// timeout!
u.Error("Timeout in Shutdown!")
return
}
}
}
func TailFile(filename string, config tail.Config, done chan bool, msgChan chan *LineEvent) {
u.Debug("Watching file ", filename, config)
t, err := tail.TailFile(filename, config)
if err != nil {
u.Error(err)
return
}
//defer func() { done <- true }()
lineHandler := MakeFileFlattener(filename, msgChan)
for line := range t.Lines {
lineHandler(line.Text)
}
err = t.Wait()
if err != nil {
u.Error(err)
}
if err := t.Stop(); err != nil {
u.Info(err)
}
}
func TestSqlParseAstCheck(t *testing.T) {
t.Parallel()
sql := `
SELECT terms(repository.description)
FROM github_member
GROUP BY repository.language, author
`
req, err := ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok := req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, len(sel.GroupBy) == 2, "has 2 group by: %v", sel.GroupBy)
gb := sel.GroupBy[0]
assert.Tf(t, gb.Expr != nil, "")
n := gb.Expr.(*expr.IdentityNode)
assert.Tf(t, n.String() == "repository.language", "%v", n)
assert.Tf(t, n.String() == "repository.language", "%v", n)
sql = `select @@version_comment limit 7`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, sel.IsLiteral(), "Should be literal? %v", sql)
assert.Tf(t, sel.Limit == 7, "has limit = 7: %v", sel.Limit)
assert.Tf(t, len(sel.Columns) == 1, "has 1 col: %v", len(sel.Columns))
assert.Tf(t, sel.Columns[0].As == "@@version_comment", "")
sql = "select `repository.full_name` from `github_public` ORDER BY `respository.full_name` asc, TOINT(`fieldname`) DESC limit 100;"
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, sel.Limit == 100, "want limit = 100 but have %v", sel.Limit)
assert.Tf(t, len(sel.OrderBy) == 2, "want 2 orderby but has %v", len(sel.OrderBy))
assert.Tf(t, sel.OrderBy[0].Order == "ASC", "%v", sel.OrderBy[0].String())
assert.Tf(t, sel.OrderBy[1].Order == "DESC", "%v", sel.OrderBy[1].String())
sql = "select `actor.id`, `actor.login` from github_watch where `actor.id` < 1000"
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, len(sel.Columns) == 2, "want 2 Columns but has %v", len(sel.Columns))
assert.Tf(t, sel.Where != nil, "where not nil?: %v", sel.Where.String())
assert.Tf(t, sel.Where.String() == "`actor.id` < 1000", "is where: %v", sel.Where.String())
// We also need to ensure that the From[].Sources are populated? Is this expected or needed?
// assert.Tf(t, len(sel.From) == 1, "Has 1 from")
// assert.Tf(t, len(sel.From[0].Columns) == 2, "wanted 2 columns in from: %#v", sel.From[0])
sql = `select repository.name, repository.stargazers from github_fork GROUP BY repository.name ORDER BY repository.stargazers DESC;`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
sql = `select repository.name, repository.stargazers
FROM github_fork
WHERE eq(repository.name,"dataux")
GROUP BY repository.name
HAVING eq(repository.name,"dataux")
ORDER BY ` + "`repository.stargazers`" + ` DESC
limit 9;`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, sel.Limit == 9, "want limit = 9 but have %v", sel.Limit)
assert.Tf(t, len(sel.OrderBy) == 1, "want 1 orderby but has %v", len(sel.OrderBy))
assert.Tf(t, sel.OrderBy[0].String() == "`repository.stargazers` DESC", "want orderby but has %v", sel.OrderBy[0].String())
// Unknown keyword SORT
sql = "select `repository.name` from github_fork SORT BY `repository.stargazers_count` DESC limit 3"
_, err = ParseSql(sql)
assert.Tf(t, err != nil, "Must fail parse: %v", err)
//assert.Tf(t, reqNil == nil, "Must fail parse: %v", reqNil)
sql = `select repository.name, respository.language, repository.stargazers
FROM github_fork
WHERE
eq(repository.name,"dataux")
AND repository.language = "go"
AND repository.name NOT LIKE "docker"
`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
//assert.Tf(t, len(sel.OrderBy) == 1, "want 1 orderby but has %v", len(sel.OrderBy))
u.Info(sel.Where.String())
sql = `
SELECT
actor, repository.name, repository.stargazers_count, repository.language
FROM github_watch
WHERE
repository.language = "go"
AND repository.forks_count > 1000
AND repository.description NOT LIKE "docker";`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
//assert.Tf(t, len(sel.OrderBy) == 1, "want 1 orderby but has %v", len(sel.OrderBy))
u.Info(sel.Where.String())
sql = `
EXPLAIN EXTENDED SELECT
actor, repository.name, repository.stargazers_count, repository.language
FROM github_watch
WHERE
repository.language = "go"
AND repository.forks_count > 1000
AND repository.description NOT LIKE "docker";`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
desc, ok := req.(*SqlDescribe)
assert.Tf(t, ok, "is SqlDescribe: %T", req)
sel, ok = desc.Stmt.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
u.Info(sel.Where.String())
// Where In Sub-Query Clause
sql = `select user_id, email
FROM mockcsv.users
WHERE user_id in
(select user_id from mockcsv.orders)`
req, err = ParseSql(sql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", sql, err)
sel, ok = req.(*SqlSelect)
assert.Tf(t, ok, "is SqlSelect: %T", req)
assert.Tf(t, len(sel.From) == 1, "has 1 from: %v", sel.From)
assert.Tf(t, sel.Where != nil && sel.Where.Source != nil, "has sub-select: %v", sel.Where)
}
func TestFilterQLAstCheck(t *testing.T) {
t.Parallel()
ql := `
FILTER
AND (
NAME != NULL,
tostring(fieldname) == "hello",
)
LIMIT 100
`
req, err := ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "expected 1 filters got:%d for %s", len(req.Filter.Filters), req.Filter.String())
cf := req.Filter.Filters[0]
assert.Tf(t, len(cf.Filter.Filters) == 2, "expected 2 filters got:%d for %s", len(cf.Filter.Filters), cf.String())
f1 := cf.Filter.Filters[0]
assert.Tf(t, f1.Expr != nil, "")
assert.Tf(t, f1.Expr.String() == "NAME != NULL", "%v", f1.Expr)
assert.Tf(t, req.Limit == 100, "wanted limit=100: %v", req.Limit)
ql = `FILTER NOT AND ( name == "bob" ) ALIAS root`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
cf = req.Filter.Filters[0]
assert.Tf(t, cf.Filter.Negate == true, "must negate")
fex := cf.Filter.Filters[0]
assert.Tf(t, fex.Expr.String() == `name == "bob"`, "Should have expr %v", fex)
assert.Tf(t, req.String() == `FILTER NOT name == "bob" ALIAS root`, "roundtrip? %v", req.String())
ql = `FILTER OR ( INCLUDE child_1, INCLUDE child_2 ) ALIAS root`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
cf = req.Filter.Filters[0]
assert.Tf(t, len(cf.Filter.Filters) == 2, "has 2 filter expr: %#v", cf.Filter.Filters)
assert.Tf(t, req.Filter.Op == lex.TokenOr, "must have or op %v", req.Filter.Op)
f1 = cf.Filter.Filters[1]
assert.Tf(t, f1.String() == `INCLUDE child_2`, "Should have include %q", f1.String())
ql = `FILTER NOT ( name == "bob", OR ( NOT INCLUDE filter_xyz , NOT exists abc ) ) ALIAS root`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
cf = req.Filter.Filters[0]
assert.Tf(t, len(cf.Filter.Filters) == 2, "has 2 filter expr: %#v", cf.Filter.Filters)
assert.Tf(t, cf.Filter.Negate == true, "must negate")
f1 = cf.Filter.Filters[1]
assert.Tf(t, len(f1.Filter.Filters) == 2, "has 2 filter subfilter: %#v", f1.String())
assert.Tf(t, f1.Filter.Op == lex.TokenOr, "is or %#v", f1.Filter.Op)
f2 := f1.Filter.Filters[0]
assert.T(t, f2.Negate == true)
assert.Tf(t, f2.String() == `NOT INCLUDE filter_xyz`, "Should have include %v", f2)
//assert.Tf(t, req.String() == ql, "roundtrip? %v", req.String())
ql = `
FILTER
AND (
-- Lets make sure the date is good
daysago(datefield) < 100
-- as well as domain
, domain(url) == "google.com"
, INCLUDE my_other_named_filter
, OR (
momentum > 20
, propensity > 50
, INCLUDE nested_filter
)
, NOT AND ( score > 20 , score < 50 )
)
ALIAS my_filter_name
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, req.Alias == "my_filter_name", "has alias: %q", req.Alias)
u.Info(req.String())
cf = req.Filter.Filters[0]
assert.Equalf(t, len(cf.Filter.Filters), 5, "expected 5 filters: %#v", cf.Filter)
f5 := cf.Filter.Filters[4]
assert.Tf(t, f5.Negate || f5.Filter.Negate, "expr negated? %s", f5.String())
assert.Tf(t, len(f5.Filter.Filters) == 2, "expr? %s", f5.String())
assert.Equal(t, f5.String(), "NOT AND ( score > 20, score < 50 )")
assert.Tf(t, len(req.Includes()) == 2, "has 2 includes: %v", req.Includes())
//assert.Equalf(t, f5.Expr.NodeType(), UnaryNodeType, "%s != %s", f5.Expr.NodeType(), UnaryNodeType)
ql = `
FILTER
AND (
-- Lets make sure the date is good
daysago(datefield) < 100
-- as well as domain
domain(url) == "google.com"
INCLUDE my_other_named_filter
OR (
momentum > 20
propensity > 50
)
NOT AND ( score > 20 , score < 50 )
)
ALIAS my_filter_name
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, req.Alias == "my_filter_name", "has alias: %q", req.Alias)
u.Info(req.String())
cf = req.Filter.Filters[0]
assert.Equalf(t, len(cf.Filter.Filters), 5, "expected 5 filters: %#v", cf.Filter)
f5 = cf.Filter.Filters[4]
assert.Tf(t, f5.Negate || f5.Filter.Negate, "expr negated? %s", f5.String())
assert.Tf(t, len(f5.Filter.Filters) == 2, "expr? %s", f5.String())
assert.Equal(t, f5.String(), "NOT AND ( score > 20, score < 50 )")
ql = `FILTER AND (
INCLUDE child_1,
INCLUDE child_2
) ALIAS root`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
cf = req.Filter.Filters[0]
for _, f := range cf.Filter.Filters {
assert.Tf(t, f.Include != "", "has include filter %q", f.String())
}
assert.Tf(t, len(cf.Filter.Filters) == 2, "want 2 filter expr: %#v", cf.Filter.Filters)
ql = `FILTER NOT INCLUDE child_1 ALIAS root`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
assert.Tf(t, req.Filter.Negate == true || req.Filter.Filters[0].Negate, "must negate %s", req.String())
fInc := cf.Filter.Filters[0]
assert.Tf(t, fInc.Include != "", "Should have include")
ql = `
FILTER *
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
fAll := req.Filter.Filters[0]
assert.Tf(t, fAll.MatchAll, "Should have match all")
ql = `
FILTER match_all
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
fAll = req.Filter.Filters[0]
assert.Tf(t, fAll.MatchAll, "Should have match all")
// Make sure we support following features
// - naked single valid expressions that are compound
// - naked expression syntax (ie, not AND())
ql = `
FILTER
NAME != NULL
AND
tostring(fieldname) == "hello"
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
ql = `
FILTER
AND (
EXISTS datefield
)
FROM user
ALIAS my_filter_name
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, req.Alias == "my_filter_name", "has alias: %q", req.Alias)
assert.Tf(t, req.From == "user", "has FROM: %q", req.From)
cf = req.Filter.Filters[0]
assert.Tf(t, len(cf.Filter.Filters) == 1, "has 1 filters: %#v", cf.Filter)
f1 = cf.Filter.Filters[0]
assert.Tf(t, f1.Expr != nil, "")
assert.Tf(t, f1.Expr.String() == "EXISTS datefield", "%#v", f1.Expr.String())
// Make sure we have a HasDateMath flag
ql = `
FILTER created > "now-3d"
`
req, err = ParseFilterQL(ql)
assert.Tf(t, err == nil && req != nil, "Must parse: %s \n\t%v", ql, err)
assert.Tf(t, len(req.Filter.Filters) == 1, "has 1 filter expr: %#v", req.Filter.Filters)
assert.Tf(t, req.HasDateMath == true, "Must recognize datemath")
}