Example #1
0
// TODO(tschottdorf): some logic tests currently take a long time to run.
// Probably a case of heartbeats timing out or many restarts in some tests.
// Need to investigate when all moving parts are in place.
func (t *logicTest) processTestFile(path string) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	defer t.traceStop()

	if t.verbose {
		fmt.Println("--- queries start here")
		defer t.printCompletion(path)
	}

	t.lastProgress = timeutil.Now()

	execKnobs := t.srv.(*server.TestServer).Cfg.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs)

	repeat := 1
	s := newLineScanner(file)
	for s.Scan() {
		if *maxErrs > 0 && t.failures >= *maxErrs {
			t.Fatalf("%s:%d: too many errors encountered, skipping the rest of the input",
				path, s.line)
		}

		fields := strings.Fields(s.Text())
		if len(fields) == 0 {
			continue
		}
		cmd := fields[0]
		if strings.HasPrefix(cmd, "#") {
			// Skip comment lines.
			continue
		}
		if len(fields) == 2 && fields[1] == "error" {
			return fmt.Errorf("%s:%d: no expected error provided", path, s.line)
		}
		switch cmd {
		case "repeat":
			// A line "repeat X" makes the test repeat the following statement or query X times.
			var err error
			count := 0
			if len(fields) != 2 {
				err = errors.New("invalid line format")
			} else if count, err = strconv.Atoi(fields[1]); err == nil && count < 2 {
				err = errors.New("invalid count")
			}
			if err != nil {
				return fmt.Errorf("%s:%d invalid repeat line: %s", path, s.line, err)
			}
			repeat = count

		case "sleep":
			var err error
			var duration time.Duration
			// A line "sleep Xs" makes the test sleep for X seconds.
			if len(fields) != 2 {
				err = errors.New("invalid line format")
			} else if duration, err = time.ParseDuration(fields[1]); err != nil {
				err = errors.New("invalid duration")
			}
			if err != nil {
				return fmt.Errorf("%s:%d invalid sleep line: %s", path, s.line, err)
			}
			time.Sleep(duration)

		case "statement":
			stmt := logicStatement{pos: fmt.Sprintf("%s:%d", path, s.line)}
			// Parse "statement error <regexp>"
			if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
				stmt.expectErrCode = m[1]
				stmt.expectErr = m[2]
			}
			var buf bytes.Buffer
			for s.Scan() {
				line := s.Text()
				if line == "" {
					break
				}
				fmt.Fprintln(&buf, line)
			}
			stmt.sql = strings.TrimSpace(buf.String())
			if !s.skip {
				for i := 0; i < repeat; i++ {
					if ok := t.execStatement(stmt); !ok {
						return fmt.Errorf("%s: error in statement, skipping to next file", stmt.pos)
					}
				}
			} else {
				s.skip = false
			}
			repeat = 1
			t.success(path)

		case "query":
			query := logicQuery{pos: fmt.Sprintf("%s:%d", path, s.line)}
			label := ""
			// Parse "query error <regexp>"
			if m := errorRE.FindStringSubmatch(s.Text()); m != nil {
				query.expectErrCode = m[1]
				query.expectErr = m[2]
			} else if len(fields) < 2 {
				return fmt.Errorf("%s: invalid test statement: %s", query.pos, s.Text())
			} else {
				// Parse "query <type-string> <sort-mode> <label>"
				// The type string specifies the number of columns and their types:
				//   - T for text
				//   - I for integer
				//   - R for floating point or decimal
				//   - B for boolean
				// The sort mode is one of:
				//   - "nosort" (default)
				//   - "rowsort"
				//   - "valuesort"
				//   - "colnames"
				//
				// The label is optional. If specified, the test runner stores a hash
				// of the results of the query under the given label. If the label is
				// reused, the test runner verifies that the results are the
				// same. This can be used to verify that two or more queries in the
				// same test script that are logically equivalent always generate the
				// same output.
				query.colTypes = fields[1]
				if len(fields) >= 3 {
					for _, opt := range strings.Split(fields[2], ",") {
						switch opt {
						case "nosort":
							query.sorter = nil

						case "rowsort":
							query.sorter = rowSort

						case "valuesort":
							query.sorter = valueSort

						case "colnames":
							query.colNames = true

						default:
							return fmt.Errorf("%s: unknown sort mode: %s", query.pos, opt)
						}
					}
				}
				if len(fields) >= 4 {
					label = fields[3]
				}
			}

			var buf bytes.Buffer
			for s.Scan() {
				line := s.Text()
				if line == "----" {
					if query.expectErr != "" {
						return fmt.Errorf("%s: invalid ---- delimiter after a query expecting an error: %s", query.pos, query.expectErr)
					}
					break
				}
				if strings.TrimSpace(s.Text()) == "" {
					break
				}
				fmt.Fprintln(&buf, line)
			}
			query.sql = strings.TrimSpace(buf.String())

			if query.expectErr == "" {
				// Query results are either a space separated list of values up to a
				// blank line or a line of the form "xx values hashing to yyy". The
				// latter format is used by sqllogictest when a large number of results
				// match the query.
				if s.Scan() {
					if m := resultsRE.FindStringSubmatch(s.Text()); m != nil {
						var err error
						query.expectedValues, err = strconv.Atoi(m[1])
						if err != nil {
							return err
						}
						query.expectedHash = m[2]
					} else {
						for {
							query.expectedResultsRaw = append(query.expectedResultsRaw, s.Text())
							results := strings.Fields(s.Text())
							if len(results) == 0 {
								break
							}
							query.expectedResults = append(query.expectedResults, results...)
							if !s.Scan() {
								break
							}
						}
						query.expectedValues = len(query.expectedResults)
					}

					if label != "" {
						expectedHash := query.expectedHash
						if expectedHash == "" {
							hash, err := t.hashResults(query.expectedResults)
							if err != nil {
								t.Error(err)
								continue
							}
							expectedHash = hash
						}
						if prevHash, ok := t.labelMap[label]; ok {
							if prevHash != expectedHash {
								t.Errorf("%s: error in input: previous reference values for label %s (hash %s) do not match new definition (hash %s)", query.pos, label, prevHash, expectedHash)
								continue
							}
						} else {
							t.labelMap[label] = expectedHash
						}
					}
				}
			}

			if !s.skip {
				for i := 0; i < repeat; i++ {
					if err := t.execQuery(query); err != nil {
						t.Error(err)
					}
				}
			} else {
				s.skip = false
			}
			repeat = 1
			t.success(path)

		case "halt", "hash-threshold":

		case "user":
			if len(fields) < 2 {
				return fmt.Errorf("user command requires one argument, found: %v", fields)
			}
			if len(fields[1]) == 0 {
				return errors.New("user command requires a non-blank argument")
			}
			cleanupUserFunc := t.setUser(fields[1])
			defer cleanupUserFunc()

		case "skipif":
			if len(fields) < 2 {
				return fmt.Errorf("skipif command requires one argument, found: %v", fields)
			}
			switch fields[1] {
			case "":
				return errors.New("skipif command requires a non-blank argument")
			case "mysql":
			case "postgresql", "cockroachdb":
				s.skip = true
				continue
			default:
				return fmt.Errorf("unimplemented test statement: %s", s.Text())
			}

		case "onlyif":
			if len(fields) < 2 {
				return fmt.Errorf("onlyif command requires one argument, found: %v", fields)
			}
			switch fields[1] {
			case "":
				return errors.New("onlyif command requires a non-blank argument")
			case "cockroachdb":
			case "mysql":
				s.skip = true
				continue
			default:
				return fmt.Errorf("unimplemented test statement: %s", s.Text())
			}

		case "traceon":
			if len(fields) != 2 {
				return fmt.Errorf("traceon requires a filename argument, found: %v", fields)
			}
			t.traceStart(fields[1])

		case "traceoff":
			if t.traceFile == nil {
				return errors.New("no trace active")
			}
			t.traceStop()

		case "fix-txn-priorities":
			// fix-txn-priorities causes future transactions to have hardcoded
			// priority values (based on the priority level), (replacing the
			// probabilistic generation).
			// The change stays in effect for the duration of that particular
			// test file.
			if len(fields) != 1 {
				return fmt.Errorf("fix-txn-priority takes no arguments, found: %v", fields[1:])
			}
			fmt.Println("Setting deterministic priorities.")

			execKnobs.FixTxnPriority = true
			defer func() { execKnobs.FixTxnPriority = false }()

		case "kv-batch-size":
			// kv-batch-size limits the kvfetcher batch size. It can be used to
			// trigger certain error conditions around limited batches.
			if len(fields) != 2 {
				return fmt.Errorf("kv-batch-size needs an integer argument, found: %v", fields[1:])
			}
			batchSize, err := strconv.Atoi(fields[1])
			if err != nil {
				return fmt.Errorf("kv-batch-size needs an integer argument; %s", err)
			}
			fmt.Printf("Setting kv batch size %d\n", batchSize)
			defer sqlbase.SetKVBatchSize(int64(batchSize))()

		default:
			return fmt.Errorf("%s:%d: unknown command: %s", path, s.line, cmd)
		}
	}

	return s.Err()
}
Example #2
0
// TestScanBatches tests the scan-in-batches code by artificially setting the batch size to
// particular values and performing queries.
func TestScanBatches(t *testing.T) {
	defer leaktest.AfterTest(t)()

	s, db, _ := serverutils.StartServer(
		t, base.TestServerArgs{UseDatabase: "test"})
	defer s.Stopper().Stop()

	if _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS test`); err != nil {
		t.Fatal(err)
	}

	// The test will screw around with KVBatchSize; make sure to restore it at the end.
	restore := sqlbase.SetKVBatchSize(10)
	defer restore()

	numAs := 5
	numBs := 20

	if _, err := db.Exec(`DROP TABLE IF EXISTS test.scan`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`CREATE TABLE test.scan (a INT, b INT, v STRING, PRIMARY KEY (a, b))`); err != nil {
		t.Fatal(err)
	}

	var buf bytes.Buffer
	buf.WriteString(`INSERT INTO test.scan VALUES `)
	for a := 0; a < numAs; a++ {
		for b := 0; b < numBs; b++ {
			if a+b > 0 {
				buf.WriteString(", ")
			}
			if (a+b)%2 == 0 {
				fmt.Fprintf(&buf, "(%d, %d, 'str%d%d')", a, b, a, b)
			} else {
				// Every other row doesn't get the string value (to have NULLs).
				fmt.Fprintf(&buf, "(%d, %d, NULL)", a, b)
			}
		}
	}
	if _, err := db.Exec(buf.String()); err != nil {
		t.Fatal(err)
	}

	// The table will have one key for the even rows, and two keys for the odd rows.
	numKeys := 3 * numAs * numBs / 2
	batchSizes := []int{1, 2, 3, 5, 10, 13, 100, numKeys - 1, numKeys, numKeys + 1}
	numSpanValues := []int{0, 1, 2, 3}

	for _, batch := range batchSizes {
		sqlbase.SetKVBatchSize(int64(batch))
		for _, numSpans := range numSpanValues {
			testScanBatchQuery(t, db, numSpans, numAs, numBs, false)
			testScanBatchQuery(t, db, numSpans, numAs, numBs, true)
		}
	}

	if _, err := db.Exec(`DROP TABLE test.scan`); err != nil {
		t.Fatal(err)
	}
}