// TODO(tschottdorf): some logic tests currently take a long time to run.
// Probably a case of heartbeats timing out or many restarts in some tests.
// Need to investigate when all moving parts are in place.
func (t *logicTest) processTestFile(path string) error {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
defer t.traceStop()
if t.verbose {
fmt.Println("--- queries start here")
defer t.printCompletion(path)
}
t.lastProgress = timeutil.Now()
execKnobs := t.srv.(*server.TestServer).Cfg.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs)
repeat := 1
s := newLineScanner(file)
for s.Scan() {
if *maxErrs > 0 && t.failures >= *maxErrs {
t.Fatalf("%s:%d: too many errors encountered, skipping the rest of the input",
path, s.line)
}
fields := strings.Fields(s.Text())
if len(fields) == 0 {
continue
}
cmd := fields[0]
if strings.HasPrefix(cmd, "#") {
// Skip comment lines.
continue
}
if len(fields) == 2 && fields[1] == "error" {
return fmt.Errorf("%s:%d: no expected error provided", path, s.line)
}
switch cmd {
case "repeat":
// A line "repeat X" makes the test repeat the following statement or query X times.
var err error
count := 0
if len(fields) != 2 {
err = errors.New("invalid line format")
} else if count, err = strconv.Atoi(fields[1]); err == nil && count < 2 {
err = errors.New("invalid count")
}
if err != nil {
return fmt.Errorf("%s:%d invalid repeat line: %s", path, s.line, err)
}
repeat = count
case "sleep":
var err error
var duration time.Duration
// A line "sleep Xs" makes the test sleep for X seconds.
if len(fields) != 2 {
err = errors.New("invalid line format")
} else if duration, err = time.ParseDuration(fields[1]); err != nil {
err = errors.New("invalid duration")
}
if err != nil {
return fmt.Errorf("%s:%d invalid sleep line: %s", path, s.line, err)
}
time.Sleep(duration)
case "statement":
stmt := logicStatement{pos: fmt.Sprintf("%s:%d", path, s.line)}
// Parse "statement error <regexp>"
if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
stmt.expectErrCode = m[1]
stmt.expectErr = m[2]
}
var buf bytes.Buffer
for s.Scan() {
line := s.Text()
if line == "" {
break
}
fmt.Fprintln(&buf, line)
}
stmt.sql = strings.TrimSpace(buf.String())
if !s.skip {
for i := 0; i < repeat; i++ {
if ok := t.execStatement(stmt); !ok {
return fmt.Errorf("%s: error in statement, skipping to next file", stmt.pos)
}
}
} else {
s.skip = false
}
repeat = 1
t.success(path)
case "query":
query := logicQuery{pos: fmt.Sprintf("%s:%d", path, s.line)}
label := ""
// Parse "query error <regexp>"
if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
query.expectErrCode = m[1]
query.expectErr = m[2]
} else if len(fields) < 2 {
return fmt.Errorf("%s: invalid test statement: %s", query.pos, s.Text())
} else {
// Parse "query <type-string> <sort-mode> <label>"
// The type string specifies the number of columns and their types:
// - T for text
// - I for integer
// - R for floating point or decimal
// - B for boolean
// The sort mode is one of:
// - "nosort" (default)
// - "rowsort"
// - "valuesort"
// - "colnames"
//
// The label is optional. If specified, the test runner stores a hash
// of the results of the query under the given label. If the label is
// reused, the test runner verifies that the results are the
// same. This can be used to verify that two or more queries in the
// same test script that are logically equivalent always generate the
// same output.
query.colTypes = fields[1]
if len(fields) >= 3 {
for _, opt := range strings.Split(fields[2], ",") {
switch opt {
case "nosort":
query.sorter = nil
case "rowsort":
query.sorter = rowSort
case "valuesort":
query.sorter = valueSort
case "colnames":
query.colNames = true
default:
return fmt.Errorf("%s: unknown sort mode: %s", query.pos, opt)
}
}
}
if len(fields) >= 4 {
label = fields[3]
}
}
var buf bytes.Buffer
for s.Scan() {
line := s.Text()
if line == "----" {
if query.expectErr != "" {
return fmt.Errorf("%s: invalid ---- delimiter after a query expecting an error: %s", query.pos, query.expectErr)
}
break
}
if strings.TrimSpace(s.Text()) == "" {
break
}
fmt.Fprintln(&buf, line)
}
query.sql = strings.TrimSpace(buf.String())
if query.expectErr == "" {
// Query results are either a space separated list of values up to a
// blank line or a line of the form "xx values hashing to yyy". The
// latter format is used by sqllogictest when a large number of results
// match the query.
if s.Scan() {
if m := resultsRE.FindStringSubmatch(s.Text()); m != nil {
var err error
query.expectedValues, err = strconv.Atoi(m[1])
if err != nil {
return err
}
query.expectedHash = m[2]
} else {
for {
query.expectedResultsRaw = append(query.expectedResultsRaw, s.Text())
results := strings.Fields(s.Text())
if len(results) == 0 {
break
}
query.expectedResults = append(query.expectedResults, results...)
if !s.Scan() {
break
}
}
query.expectedValues = len(query.expectedResults)
}
if label != "" {
expectedHash := query.expectedHash
if expectedHash == "" {
hash, err := t.hashResults(query.expectedResults)
if err != nil {
t.Error(err)
continue
}
expectedHash = hash
}
if prevHash, ok := t.labelMap[label]; ok {
if prevHash != expectedHash {
t.Errorf("%s: error in input: previous reference values for label %s (hash %s) do not match new definition (hash %s)", query.pos, label, prevHash, expectedHash)
continue
}
} else {
t.labelMap[label] = expectedHash
}
}
}
}
if !s.skip {
for i := 0; i < repeat; i++ {
if err := t.execQuery(query); err != nil {
t.Error(err)
}
}
} else {
s.skip = false
}
repeat = 1
t.success(path)
case "halt", "hash-threshold":
case "user":
if len(fields) < 2 {
return fmt.Errorf("user command requires one argument, found: %v", fields)
}
if len(fields[1]) == 0 {
return errors.New("user command requires a non-blank argument")
}
cleanupUserFunc := t.setUser(fields[1])
defer cleanupUserFunc()
case "skipif":
if len(fields) < 2 {
return fmt.Errorf("skipif command requires one argument, found: %v", fields)
}
switch fields[1] {
case "":
return errors.New("skipif command requires a non-blank argument")
case "mysql":
case "postgresql", "cockroachdb":
s.skip = true
continue
default:
return fmt.Errorf("unimplemented test statement: %s", s.Text())
}
case "onlyif":
if len(fields) < 2 {
return fmt.Errorf("onlyif command requires one argument, found: %v", fields)
}
switch fields[1] {
case "":
return errors.New("onlyif command requires a non-blank argument")
case "cockroachdb":
case "mysql":
s.skip = true
continue
default:
return fmt.Errorf("unimplemented test statement: %s", s.Text())
}
case "traceon":
if len(fields) != 2 {
return fmt.Errorf("traceon requires a filename argument, found: %v", fields)
}
t.traceStart(fields[1])
case "traceoff":
if t.traceFile == nil {
return errors.New("no trace active")
}
t.traceStop()
case "fix-txn-priorities":
// fix-txn-priorities causes future transactions to have hardcoded
// priority values (based on the priority level), (replacing the
// probabilistic generation).
// The change stays in effect for the duration of that particular
// test file.
if len(fields) != 1 {
return fmt.Errorf("fix-txn-priority takes no arguments, found: %v", fields[1:])
}
fmt.Println("Setting deterministic priorities.")
execKnobs.FixTxnPriority = true
defer func() { execKnobs.FixTxnPriority = false }()
case "kv-batch-size":
// kv-batch-size limits the kvfetcher batch size. It can be used to
// trigger certain error conditions around limited batches.
if len(fields) != 2 {
return fmt.Errorf("kv-batch-size needs an integer argument, found: %v", fields[1:])
}
batchSize, err := strconv.Atoi(fields[1])
if err != nil {
return fmt.Errorf("kv-batch-size needs an integer argument; %s", err)
}
fmt.Printf("Setting kv batch size %d\n", batchSize)
defer sqlbase.SetKVBatchSize(int64(batchSize))()
default:
return fmt.Errorf("%s:%d: unknown command: %s", path, s.line, cmd)
}
}
return s.Err()
}
// TestScanBatches tests the scan-in-batches code by artificially setting the batch size to
// particular values and performing queries.
func TestScanBatches(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db, _ := serverutils.StartServer(
t, base.TestServerArgs{UseDatabase: "test"})
defer s.Stopper().Stop()
if _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS test`); err != nil {
t.Fatal(err)
}
// The test will screw around with KVBatchSize; make sure to restore it at the end.
restore := sqlbase.SetKVBatchSize(10)
defer restore()
numAs := 5
numBs := 20
if _, err := db.Exec(`DROP TABLE IF EXISTS test.scan`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`CREATE TABLE test.scan (a INT, b INT, v STRING, PRIMARY KEY (a, b))`); err != nil {
t.Fatal(err)
}
var buf bytes.Buffer
buf.WriteString(`INSERT INTO test.scan VALUES `)
for a := 0; a < numAs; a++ {
for b := 0; b < numBs; b++ {
if a+b > 0 {
buf.WriteString(", ")
}
if (a+b)%2 == 0 {
fmt.Fprintf(&buf, "(%d, %d, 'str%d%d')", a, b, a, b)
} else {
// Every other row doesn't get the string value (to have NULLs).
fmt.Fprintf(&buf, "(%d, %d, NULL)", a, b)
}
}
}
if _, err := db.Exec(buf.String()); err != nil {
t.Fatal(err)
}
// The table will have one key for the even rows, and two keys for the odd rows.
numKeys := 3 * numAs * numBs / 2
batchSizes := []int{1, 2, 3, 5, 10, 13, 100, numKeys - 1, numKeys, numKeys + 1}
numSpanValues := []int{0, 1, 2, 3}
for _, batch := range batchSizes {
sqlbase.SetKVBatchSize(int64(batch))
for _, numSpans := range numSpanValues {
testScanBatchQuery(t, db, numSpans, numAs, numBs, false)
testScanBatchQuery(t, db, numSpans, numAs, numBs, true)
}
}
if _, err := db.Exec(`DROP TABLE test.scan`); err != nil {
t.Fatal(err)
}
}