func (s *testColumnChangeSuite) checkAddPublic(d *ddl, ctx context.Context, writeOnlyTable, publicTable table.Table) error { // publicTable Insert t values (4, 4, 4) h, err := publicTable.AddRecord(ctx, types.MakeDatums(4, 4, 4)) if err != nil { return errors.Trace(err) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } // writeOnlyTable update t set c1 = 3 where c1 = 4 oldRow, err := writeOnlyTable.RowWithCols(ctx, h, writeOnlyTable.WritableCols()) if err != nil { return errors.Trace(err) } if len(oldRow) != 3 { return errors.Errorf("%v", oldRow) } newRow := types.MakeDatums(3, 4, oldRow[2].GetValue()) err = writeOnlyTable.UpdateRecord(ctx, h, oldRow, newRow, touchedMap(writeOnlyTable)) if err != nil { return errors.Trace(err) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } // publicTable select * from t, make sure the new c3 value 4 is not overwritten to default value 3. err = checkResult(ctx, publicTable, testutil.RowsWithSep(" ", "2 3 3", "3 4 4")) if err != nil { return errors.Trace(err) } return nil }
func runStmt(ctx context.Context, s ast.Statement, args ...interface{}) (ast.RecordSet, error) { var err error var rs ast.RecordSet // before every execution, we must clear affectedrows. variable.GetSessionVars(ctx).SetAffectedRows(0) if s.IsDDL() { err = ctx.CommitTxn() if err != nil { return nil, errors.Trace(err) } } rs, err = s.Exec(ctx) // All the history should be added here. se := ctx.(*session) se.history.add(0, s) // MySQL DDL should be auto-commit. ac, err1 := autocommit.ShouldAutocommit(ctx) if err1 != nil { return nil, errors.Trace(err1) } if s.IsDDL() || ac { if err != nil { log.Info("RollbackTxn for ddl/autocommit error.") ctx.RollbackTxn() } else { err = ctx.CommitTxn() } } return rs, errors.Trace(err) }
func (s *testColumnSuite) checkColumnKVExist(ctx context.Context, t table.Table, handle int64, col *table.Column, columnValue interface{}, isExist bool) error { txn, err := ctx.GetTxn(true) if err != nil { return errors.Trace(err) } defer ctx.CommitTxn() key := t.RecordKey(handle) data, err := txn.Get(key) if !isExist { if terror.ErrorEqual(err, kv.ErrNotExist) { return nil } } if err != nil { return errors.Trace(err) } colMap := make(map[int64]*types.FieldType) colMap[col.ID] = &col.FieldType rowMap, err := tablecodec.DecodeRow(data, colMap) if err != nil { return errors.Trace(err) } val, ok := rowMap[col.ID] if isExist { if !ok || val.GetValue() != columnValue { return errors.Errorf("%v is not equal to %v", val.GetValue(), columnValue) } } else { if ok { return errors.Errorf("column value should not exists") } } return nil }
func (d *ddl) doDDLJob(ctx context.Context, job *model.Job) error { // for every DDL, we must commit current transaction. if err := ctx.CommitTxn(); err != nil { return errors.Trace(err) } // Create a new job and queue it. err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { t := meta.NewMeta(txn) var err error job.ID, err = t.GenGlobalID() if err != nil { return errors.Trace(err) } err = t.EnQueueDDLJob(job) return errors.Trace(err) }) if err != nil { return errors.Trace(err) } // notice worker that we push a new job and wait the job done. asyncNotify(d.ddlJobCh) log.Warnf("[ddl] start DDL job %v", job) jobID := job.ID var historyJob *model.Job // for a job from start to end, the state of it will be none -> delete only -> write only -> reorganization -> public // for every state changes, we will wait as lease 2 * lease time, so here the ticker check is 10 * lease. ticker := time.NewTicker(chooseLeaseTime(10*d.lease, 10*time.Second)) defer ticker.Stop() for { select { case <-d.ddlJobDoneCh: case <-ticker.C: } historyJob, err = d.getHistoryDDLJob(jobID) if err != nil { log.Errorf("[ddl] get history DDL job err %v, check again", err) continue } else if historyJob == nil { log.Warnf("[ddl] DDL job %d is not in history, maybe not run", jobID) continue } // if a job is a history table, the state must be JobDone or JobCancel. if historyJob.State == model.JobDone { return nil } return errors.Errorf(historyJob.Error) } }
func (s *testColumnSuite) checkPublicColumn(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, col *table.Column, row []types.Datum, columnValue interface{}) { t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) _, err := ctx.GetTxn(true) c.Assert(err, IsNil) i := int64(0) oldRow := append(row, types.NewDatum(columnValue)) err = t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, oldRow) i++ return true, nil }) c.Assert(err, IsNil) c.Assert(i, Equals, int64(1)) // Test add a new row. _, err = ctx.GetTxn(true) c.Assert(err, IsNil) newRow := types.MakeDatums(int64(11), int64(22), int64(33), int64(44)) handle, err = t.AddRecord(ctx, newRow) c.Assert(err, IsNil) _, err = ctx.GetTxn(true) c.Assert(err, IsNil) rows := [][]types.Datum{oldRow, newRow} i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, rows[i]) i++ return true, nil }) c.Assert(i, Equals, int64(2)) // Test remove a row. _, err = ctx.GetTxn(true) c.Assert(err, IsNil) err = t.RemoveRecord(ctx, handle, newRow) c.Assert(err, IsNil) _, err = ctx.GetTxn(true) c.Assert(err, IsNil) i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, oldRow) i++ return true, nil }) c.Assert(i, Equals, int64(1)) err = ctx.CommitTxn() c.Assert(err, IsNil) s.testGetColumn(c, t, col.Name.L, true) }
func (d *ddl) doDDLJob(ctx context.Context, job *model.Job) error { // For every DDL, we must commit current transaction. if err := ctx.CommitTxn(); err != nil { return errors.Trace(err) } // Get a global job ID and put the DDL job in the queue. err := d.addDDLJob(ctx, job) if err != nil { return errors.Trace(err) } // Notice worker that we push a new job and wait the job done. asyncNotify(d.ddlJobCh) log.Infof("[ddl] start DDL job %s", job) var historyJob *model.Job jobID := job.ID // For a job from start to end, the state of it will be none -> delete only -> write only -> reorganization -> public // For every state changes, we will wait as lease 2 * lease time, so here the ticker check is 10 * lease. ticker := time.NewTicker(chooseLeaseTime(10*d.lease, 10*time.Second)) startTime := time.Now() jobsGauge.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String()).Inc() defer func() { ticker.Stop() jobsGauge.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String()).Dec() retLabel := handleJobSucc if err != nil { retLabel = handleJobFailed } handleJobHistogram.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String(), retLabel).Observe(time.Since(startTime).Seconds()) }() for { select { case <-d.ddlJobDoneCh: case <-ticker.C: } historyJob, err = d.getHistoryDDLJob(jobID) if err != nil { log.Errorf("[ddl] get history DDL job err %v, check again", err) continue } else if historyJob == nil { log.Warnf("[ddl] DDL job %d is not in history, maybe not run", jobID) continue } // If a job is a history job, the state must be JobDone or JobCancel. if historyJob.State == model.JobDone { log.Infof("[ddl] DDL job %d is finished", jobID) return nil } return errors.Trace(historyJob.Error) } }
func (s *testIndexSuite) checkIndexKVExist(c *C, ctx context.Context, t table.Table, handle int64, indexCol table.Index, columnValues []types.Datum, isExist bool) { c.Assert(len(indexCol.Meta().Columns), Equals, len(columnValues)) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) exist, _, err := indexCol.Exist(txn, columnValues, handle) c.Assert(err, IsNil) c.Assert(exist, Equals, isExist) err = ctx.CommitTxn() c.Assert(err, IsNil) }
func checkCases(cases []testCase, ld *executor.LoadDataInfo, c *C, tk *testkit.TestKit, ctx context.Context, selectSQL, deleteSQL string) { for _, ca := range cases { data, err1 := ld.InsertData(ca.data1, ca.data2) c.Assert(err1, IsNil) if ca.restData == nil { c.Assert(data, HasLen, 0, Commentf("data1:%v, data2:%v, data:%v", string(ca.data1), string(ca.data2), string(data))) } else { c.Assert(data, DeepEquals, ca.restData, Commentf("data1:%v, data2:%v, data:%v", string(ca.data1), string(ca.data2), string(data))) } err1 = ctx.CommitTxn() c.Assert(err1, IsNil) r := tk.MustQuery(selectSQL) r.Check(testkit.Rows(ca.expected...)) tk.MustExec(deleteSQL) } }
func (s *testColumnSuite) checkColumnKVExist(c *C, ctx context.Context, t table.Table, handle int64, col *table.Column, columnValue interface{}, isExist bool) { txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) key := t.RecordKey(handle, col) data, err := txn.Get(key) if isExist { c.Assert(err, IsNil) v, err1 := tables.DecodeValue(data, &col.FieldType) c.Assert(err1, IsNil) value, err1 := v.ConvertTo(&col.FieldType) c.Assert(err1, IsNil) c.Assert(value.GetValue(), Equals, columnValue) } else { c.Assert(err, NotNil) } err = ctx.CommitTxn() c.Assert(err, IsNil) }
// runStmt executes the ast.Statement and commit or rollback the current transaction. func runStmt(ctx context.Context, s ast.Statement) (ast.RecordSet, error) { var err error var rs ast.RecordSet if s.IsDDL() { err = ctx.CommitTxn() if err != nil { return nil, errors.Trace(err) } } rs, err = s.Exec(ctx) // All the history should be added here. se := ctx.(*session) getHistory(ctx).add(0, s) // MySQL DDL should be auto-commit. if s.IsDDL() || se.sessionVars.ShouldAutocommit() { if err != nil { log.Info("RollbackTxn for ddl/autocommit error.") ctx.RollbackTxn() } else { err = ctx.CommitTxn() } } return rs, errors.Trace(err) }
func (s *testColumnChangeSuite) checkAddWriteOnly(d *ddl, ctx context.Context, deleteOnlyTable, writeOnlyTable table.Table) error { // WriteOnlyTable: insert t values (2, 3) _, err := writeOnlyTable.AddRecord(ctx, types.MakeDatums(2, 3)) if err != nil { return errors.Trace(err) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } err = checkResult(ctx, writeOnlyTable, testutil.RowsWithSep(" ", "1 2 <nil>", "2 3 3")) if err != nil { return errors.Trace(err) } // DeleteOnlyTable: select * from t err = checkResult(ctx, deleteOnlyTable, testutil.RowsWithSep(" ", "1 2", "2 3")) if err != nil { return errors.Trace(err) } // WriteOnlyTable: update t set c1 = 2 where c1 = 1 h, _, err := writeOnlyTable.Seek(ctx, 0) if err != nil { return errors.Trace(err) } err = writeOnlyTable.UpdateRecord(ctx, h, types.MakeDatums(1, 2), types.MakeDatums(2, 2), touchedMap(writeOnlyTable)) if err != nil { return errors.Trace(err) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } // After we update the first row, its default value is also set. err = checkResult(ctx, writeOnlyTable, testutil.RowsWithSep(" ", "2 2 3", "2 3 3")) if err != nil { return errors.Trace(err) } // DeleteOnlyTable: delete from t where c2 = 2 err = deleteOnlyTable.RemoveRecord(ctx, h, types.MakeDatums(2, 2)) if err != nil { return errors.Trace(err) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } // After delete table has deleted the first row, check the WriteOnly table records. err = checkResult(ctx, writeOnlyTable, testutil.RowsWithSep(" ", "2 3 3")) return errors.Trace(err) }
func (s *testColumnSuite) checkPublicColumn(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, newCol *table.Column, oldRow []types.Datum, columnValue interface{}) error { t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) _, err := ctx.GetTxn(true) if err != nil { return errors.Trace(err) } i := int64(0) updatedRow := append(oldRow, types.NewDatum(columnValue)) err = t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { if !reflect.DeepEqual(data, updatedRow) { return false, errors.Errorf("%v not equal to %v", data, updatedRow) } i++ return true, nil }) if err != nil { return errors.Trace(err) } if i != 1 { return errors.Errorf("expect 1, got %v", i) } // Test add a new row. _, err = ctx.GetTxn(true) if err != nil { return errors.Trace(err) } newRow := types.MakeDatums(int64(11), int64(22), int64(33), int64(44)) handle, err = t.AddRecord(ctx, newRow) if err != nil { return errors.Trace(err) } _, err = ctx.GetTxn(true) if err != nil { return errors.Trace(err) } rows := [][]types.Datum{updatedRow, newRow} i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { if !reflect.DeepEqual(data, rows[i]) { return false, errors.Errorf("%v not equal to %v", data, rows[i]) } i++ return true, nil }) if i != 2 { return errors.Errorf("expect 2, got %v", i) } // Test remove a row. _, err = ctx.GetTxn(true) if err != nil { return errors.Trace(err) } err = t.RemoveRecord(ctx, handle, newRow) if err != nil { return errors.Trace(err) } _, err = ctx.GetTxn(true) if err != nil { return errors.Trace(err) } i = int64(0) err = t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, updatedRow) i++ return true, nil }) if err != nil { return errors.Trace(err) } if i != 1 { return errors.Errorf("expect 1, got %v", i) } err = ctx.CommitTxn() if err != nil { return errors.Trace(err) } err = s.testGetColumn(t, newCol.Name.L, true) if err != nil { return errors.Trace(err) } return nil }
func (s *testColumnSuite) TestColumn(c *C) { defer testleak.AfterTest(c)() tblInfo := testTableInfo(c, s.d, "t1", 3) var ctx context.Context ctx = testNewContext(c, s.d) defer ctx.RollbackTxn() testCreateTable(c, ctx, s.d, s.dbInfo, tblInfo) t := testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) num := 10 for i := 0; i < num; i++ { _, err := t.AddRecord(ctx, types.MakeDatums(i, 10*i, 100*i)) c.Assert(err, IsNil) } err := ctx.CommitTxn() c.Assert(err, IsNil) i := int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, HasLen, 3) c.Assert(data[0].GetInt64(), Equals, i) c.Assert(data[1].GetInt64(), Equals, 10*i) c.Assert(data[2].GetInt64(), Equals, 100*i) i++ return true, nil }) c.Assert(i, Equals, int64(num)) c.Assert(table.FindCol(t.Cols(), "c4"), IsNil) job := testCreateColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c4", &ast.ColumnPosition{Tp: ast.ColumnPositionAfter, RelativeColumn: &ast.ColumnName{Name: model.NewCIStr("c3")}}, 100) testCheckJobDone(c, s.d, job, true) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) c.Assert(table.FindCol(t.Cols(), "c4"), NotNil) i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, HasLen, 4) c.Assert(data[0].GetInt64(), Equals, i) c.Assert(data[1].GetInt64(), Equals, 10*i) c.Assert(data[2].GetInt64(), Equals, 100*i) c.Assert(data[3].GetInt64(), Equals, int64(100)) i++ return true, nil }) c.Assert(i, Equals, int64(num)) h, err := t.AddRecord(ctx, types.MakeDatums(11, 12, 13, 14)) c.Assert(err, IsNil) err = ctx.CommitTxn() c.Assert(err, IsNil) values, err := t.RowWithCols(ctx, h, t.Cols()) c.Assert(err, IsNil) c.Assert(values, HasLen, 4) c.Assert(values[3].GetInt64(), Equals, int64(14)) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c4", false) testCheckJobDone(c, s.d, job, false) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) values, err = t.RowWithCols(ctx, h, t.Cols()) c.Assert(err, IsNil) c.Assert(values, HasLen, 3) c.Assert(values[2].GetInt64(), Equals, int64(13)) job = testCreateColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c4", &ast.ColumnPosition{Tp: ast.ColumnPositionNone}, 111) testCheckJobDone(c, s.d, job, true) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) values, err = t.RowWithCols(ctx, h, t.Cols()) c.Assert(err, IsNil) c.Assert(values, HasLen, 4) c.Assert(values[3].GetInt64(), Equals, int64(111)) job = testCreateColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c5", &ast.ColumnPosition{Tp: ast.ColumnPositionNone}, 101) testCheckJobDone(c, s.d, job, true) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) values, err = t.RowWithCols(ctx, h, t.Cols()) c.Assert(err, IsNil) c.Assert(values, HasLen, 5) c.Assert(values[4].GetInt64(), Equals, int64(101)) job = testCreateColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c6", &ast.ColumnPosition{Tp: ast.ColumnPositionFirst}, 202) testCheckJobDone(c, s.d, job, true) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) cols := t.Cols() c.Assert(cols, HasLen, 6) c.Assert(cols[0].Offset, Equals, 0) c.Assert(cols[0].Name.L, Equals, "c6") c.Assert(cols[1].Offset, Equals, 1) c.Assert(cols[1].Name.L, Equals, "c1") c.Assert(cols[2].Offset, Equals, 2) c.Assert(cols[2].Name.L, Equals, "c2") c.Assert(cols[3].Offset, Equals, 3) c.Assert(cols[3].Name.L, Equals, "c3") c.Assert(cols[4].Offset, Equals, 4) c.Assert(cols[4].Name.L, Equals, "c4") c.Assert(cols[5].Offset, Equals, 5) c.Assert(cols[5].Name.L, Equals, "c5") values, err = t.RowWithCols(ctx, h, cols) c.Assert(err, IsNil) c.Assert(values, HasLen, 6) c.Assert(values[0].GetInt64(), Equals, int64(202)) c.Assert(values[5].GetInt64(), Equals, int64(101)) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c2", false) testCheckJobDone(c, s.d, job, false) t = testGetTable(c, s.d, s.dbInfo.ID, tblInfo.ID) values, err = t.RowWithCols(ctx, h, t.Cols()) c.Assert(err, IsNil) c.Assert(values, HasLen, 5) c.Assert(values[0].GetInt64(), Equals, int64(202)) c.Assert(values[4].GetInt64(), Equals, int64(101)) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c1", false) testCheckJobDone(c, s.d, job, false) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c3", false) testCheckJobDone(c, s.d, job, false) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c4", false) testCheckJobDone(c, s.d, job, false) job = testCreateIndex(c, ctx, s.d, s.dbInfo, tblInfo, false, "c5_idx", "c5") testCheckJobDone(c, s.d, job, true) testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c5", true) testDropIndex(c, ctx, s.d, s.dbInfo, tblInfo, "c5_idx") testCheckJobDone(c, s.d, job, true) job = testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c5", false) testCheckJobDone(c, s.d, job, false) testDropColumn(c, ctx, s.d, s.dbInfo, tblInfo, "c6", true) testDropTable(c, ctx, s.d, s.dbInfo, tblInfo) }
func (d *ddl) doDDLJob(ctx context.Context, job *model.Job) error { // for every DDL, we must commit current transaction. if err := ctx.CommitTxn(); err != nil { return errors.Trace(err) } var startTS uint64 err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { t := meta.NewMeta(txn) var err error job.ID, err = t.GenGlobalID() startTS = txn.StartTS() return errors.Trace(err) }) if err != nil { return errors.Trace(err) } ddlQuery, _ := ctx.Value(context.QueryString).(string) job.Query = ddlQuery // Create a new job and queue it. err = kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { t := meta.NewMeta(txn) err1 := t.EnQueueDDLJob(job) return errors.Trace(err1) }) if err != nil { return errors.Trace(err) } // notice worker that we push a new job and wait the job done. asyncNotify(d.ddlJobCh) log.Warnf("[ddl] start DDL job %v", job) var historyJob *model.Job jobID := job.ID // for a job from start to end, the state of it will be none -> delete only -> write only -> reorganization -> public // for every state changes, we will wait as lease 2 * lease time, so here the ticker check is 10 * lease. ticker := time.NewTicker(chooseLeaseTime(10*d.lease, 10*time.Second)) startTime := time.Now() jobsGauge.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String()).Inc() defer func() { ticker.Stop() jobsGauge.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String()).Dec() retLabel := handleJobSucc if err != nil { retLabel = handleJobFailed } handleJobHistogram.WithLabelValues(JobType(ddlJobFlag).String(), job.Type.String(), retLabel).Observe(time.Since(startTime).Seconds()) }() for { select { case <-d.ddlJobDoneCh: case <-ticker.C: } historyJob, err = d.getHistoryDDLJob(jobID) if err != nil { log.Errorf("[ddl] get history DDL job err %v, check again", err) continue } else if historyJob == nil { log.Warnf("[ddl] DDL job %d is not in history, maybe not run", jobID) continue } // if a job is a history table, the state must be JobDone or JobCancel. if historyJob.State == model.JobDone { return nil } return errors.Trace(historyJob.Error) } }