// backfillIndexInTxn deals with a part of backfilling index data in a Transaction. // This part of the index data rows is defaultSmallBatchCnt. func (d *ddl) backfillIndexInTxn(t table.Table, kvIdx table.Index, handles []int64, txn kv.Transaction) (int64, error) { idxRecords, err := d.fetchRowColVals(txn, t, handles, kvIdx.Meta()) if err != nil { return 0, errors.Trace(err) } for _, idxRecord := range idxRecords { log.Debug("[ddl] backfill index...", idxRecord.handle) err = txn.LockKeys(idxRecord.key) if err != nil { return 0, errors.Trace(err) } // Create the index. handle, err := kvIdx.Create(txn, idxRecord.vals, idxRecord.handle) if err != nil { if terror.ErrorEqual(err, kv.ErrKeyExists) && idxRecord.handle == handle { // Index already exists, skip it. continue } return 0, errors.Trace(err) } } return idxRecords[len(idxRecords)-1].handle, nil }
func checkRecordAndIndex(txn kv.Transaction, t table.Table, idx table.Index) error { cols := make([]*table.Column, len(idx.Meta().Columns)) for i, col := range idx.Meta().Columns { cols[i] = t.Cols()[col.Offset] } startKey := t.RecordKey(0, nil) filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) { isExist, h2, err := idx.Exist(txn, vals1, h1) if terror.ErrorEqual(err, kv.ErrKeyExists) { record1 := &RecordData{Handle: h1, Values: vals1} record2 := &RecordData{Handle: h2, Values: vals1} return false, errDateNotEqual.Gen("index:%v != record:%v", record2, record1) } if err != nil { return false, errors.Trace(err) } if !isExist { record := &RecordData{Handle: h1, Values: vals1} return false, errDateNotEqual.Gen("index:%v != record:%v", nil, record) } return true, nil } err := iterRecords(txn, t, startKey, cols, filterFunc) if err != nil { return errors.Trace(err) } return nil }
// BuildIndexForRow implements table.Table BuildIndexForRow interface. func (t *Table) buildIndexForRow(rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index) error { if idx.Meta().State == model.StateDeleteOnly || idx.Meta().State == model.StateDeleteReorganization { // If the index is in delete only or write reorganization state, we can not add index. return nil } if _, err := idx.Create(rm, vals, h); err != nil { return errors.Trace(err) } return nil }
func (s *testIndexSuite) checkIndexKVExist(c *C, ctx context.Context, t table.Table, handle int64, indexCol table.Index, columnValues []types.Datum, isExist bool) { c.Assert(len(indexCol.Meta().Columns), Equals, len(columnValues)) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) exist, _, err := indexCol.Exist(txn, columnValues, handle) c.Assert(err, IsNil) c.Assert(exist, Equals, isExist) err = ctx.CommitTxn() c.Assert(err, IsNil) }
func checkIndexAndRecord(txn kv.Transaction, t table.Table, idx table.Index) error { it, err := idx.SeekFirst(txn) if err != nil { return errors.Trace(err) } defer it.Close() cols := make([]*table.Column, len(idx.Meta().Columns)) for i, col := range idx.Meta().Columns { cols[i] = t.Cols()[col.Offset] } for { vals1, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } else if err != nil { return errors.Trace(err) } vals2, err := rowWithCols(txn, t, h, cols) if terror.ErrorEqual(err, kv.ErrNotExist) { record := &RecordData{Handle: h, Values: vals1} err = errDateNotEqual.Gen("index:%v != record:%v", record, nil) } if err != nil { return errors.Trace(err) } if !reflect.DeepEqual(vals1, vals2) { record1 := &RecordData{Handle: h, Values: vals1} record2 := &RecordData{Handle: h, Values: vals2} return errDateNotEqual.Gen("index:%v != record:%v", record1, record2) } } return nil }
func (s *testIndexSuite) checkNoneIndex(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, index table.Index, row []types.Datum) { t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) columnValues := make([]types.Datum, len(index.Meta().Columns)) for i, column := range index.Meta().Columns { columnValues[i] = row[column.Offset] } s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false) s.testGetIndex(c, t, index.Meta().Columns[0].Name.L, false) }
func (s *testDBSuite) testAddIndex(c *C) { done := make(chan struct{}, 1) num := 100 // first add some rows for i := 0; i < num; i++ { s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } go func() { s.mustExec(c, "create index c3_index on t1 (c3)") done <- struct{}{} }() deletedKeys := make(map[int]struct{}) ticker := time.NewTicker(s.lease / 2) defer ticker.Stop() LOOP: for { select { case <-done: break LOOP case <-ticker.C: step := 10 // delete some rows, and add some data for i := num; i < num+step; i++ { n := rand.Intn(num) deletedKeys[n] = struct{}{} s.mustExec(c, "delete from t1 where c1 = ?", n) s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } num += step } } // get exists keys keys := make([]int, 0, num) for i := 0; i < num; i++ { if _, ok := deletedKeys[i]; ok { continue } keys = append(keys, i) } // test index key for _, key := range keys { rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key) matchRows(c, rows, [][]interface{}{{key}}) } // test delete key not in index for key := range deletedKeys { rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key) matchRows(c, rows, nil) } // test index range for i := 0; i < 100; i++ { index := rand.Intn(len(keys) - 3) rows := s.mustQuery(c, "select c1 from t1 where c3 >= ? limit 3", keys[index]) matchRows(c, rows, [][]interface{}{{keys[index]}, {keys[index+1]}, {keys[index+2]}}) } // TODO: support explain in future. //rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 100") //ay := dumpRows(c, rows) //c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsTrue) // get all row handles ctx := s.s.(context.Context) t := s.testGetTable(c, "t1") handles := make(map[int64]struct{}) err := t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { handles[h] = struct{}{} return true, nil }) c.Assert(err, IsNil) // check in index var nidx table.Index for _, tidx := range t.Indices() { if tidx.Meta().Name.L == "c3_index" { nidx = tidx break } } // Make sure there is index with name c3_index c.Assert(nidx, NotNil) c.Assert(nidx.Meta().ID, Greater, int64(0)) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) defer ctx.RollbackTxn() it, err := nidx.SeekFirst(txn) c.Assert(err, IsNil) defer it.Close() for { _, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } c.Assert(err, IsNil) _, ok := handles[h] c.Assert(ok, IsTrue) delete(handles, h) } c.Assert(handles, HasLen, 0) }
func (s *testDBSuite) testDropIndex(c *C) { done := make(chan struct{}, 1) s.mustExec(c, "delete from t1") num := 100 // add some rows for i := 0; i < num; i++ { s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } t := s.testGetTable(c, "t1") var c3idx table.Index for _, tidx := range t.Indices() { if tidx.Meta().Name.L == "c3_index" { c3idx = tidx break } } c.Assert(c3idx, NotNil) go func() { s.mustExec(c, "drop index c3_index on t1") done <- struct{}{} }() ticker := time.NewTicker(s.lease / 2) defer ticker.Stop() LOOP: for { select { case <-done: break LOOP case <-ticker.C: step := 10 // delete some rows, and add some data for i := num; i < num+step; i++ { n := rand.Intn(num) s.mustExec(c, "update t1 set c2 = 1 where c1 = ?", n) s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } num += step } } rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 0") ay := dumpRows(c, rows) c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsFalse) // check in index, must no index in kv ctx := s.s.(context.Context) handles := make(map[int64]struct{}) t = s.testGetTable(c, "t1") var nidx table.Index for _, tidx := range t.Indices() { if tidx.Meta().Name.L == "c3_index" { nidx = tidx break } } // Make sure there is no index with name c3_index c.Assert(nidx, IsNil) idx := tables.NewIndex(t.Meta(), c3idx.Meta()) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) defer ctx.RollbackTxn() it, err := idx.SeekFirst(txn) c.Assert(err, IsNil) defer it.Close() for { _, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } c.Assert(err, IsNil) handles[h] = struct{}{} } c.Assert(handles, HasLen, 0) }
func (s *testSuite) testIndex(c *C, tb table.Table, idx table.Index) { txn, err := s.store.Begin() c.Assert(err, IsNil) err = CompareIndexData(txn, tb, idx) c.Assert(err, IsNil) cnt, err := GetIndexRecordsCount(txn, idx, nil) c.Assert(err, IsNil) c.Assert(cnt, Equals, int64(2)) // current index data: // index data (handle, data): (1, 10), (2, 20), (3, 30) // index col data (handle, data): (1, 10), (2, 20), (4, 40) err = idx.Create(txn, types.MakeDatums(int64(30)), 3) c.Assert(err, IsNil) col := tb.Cols()[idx.Meta().Columns[0].Offset] key := tb.RecordKey(4, col) err = tables.SetColValue(txn, key, types.NewDatum(int64(40))) c.Assert(err, IsNil) err = txn.Commit() c.Assert(err, IsNil) txn, err = s.store.Begin() c.Assert(err, IsNil) err = CompareIndexData(txn, tb, idx) c.Assert(err, NotNil) record1 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(30))} diffMsg := newDiffRetError("index", record1, &RecordData{Handle: int64(3), Values: types.MakeDatums(nil)}) c.Assert(err.Error(), DeepEquals, diffMsg) // current index data: // index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) // index col data (handle, data): (1, 10), (2, 20), (4, 40), (3, 31) err = idx.Create(txn, types.MakeDatums(int64(40)), 4) c.Assert(err, IsNil) key = tb.RecordKey(3, col) err = tables.SetColValue(txn, key, types.NewDatum(int64(31))) c.Assert(err, IsNil) err = txn.Commit() c.Assert(err, IsNil) txn, err = s.store.Begin() c.Assert(err, IsNil) err = CompareIndexData(txn, tb, idx) c.Assert(err, NotNil) record2 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(31))} diffMsg = newDiffRetError("index", record1, record2) c.Assert(err.Error(), DeepEquals, diffMsg) // current index data: // index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) // index col data (handle, data): (1, 10), (2, 20), (4, 40), (5, 30) key = tb.RecordKey(3, col) txn.Delete(key) key = tb.RecordKey(5, col) err = tables.SetColValue(txn, key, types.NewDatum(int64(30))) c.Assert(err, IsNil) err = txn.Commit() c.Assert(err, IsNil) txn, err = s.store.Begin() c.Assert(err, IsNil) err = checkRecordAndIndex(txn, tb, idx) c.Assert(err, NotNil) record2 = &RecordData{Handle: int64(5), Values: types.MakeDatums(int64(30))} diffMsg = newDiffRetError("index", record1, record2) c.Assert(err.Error(), DeepEquals, diffMsg) // current index data: // index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) // index col data (handle, data): (1, 10), (2, 20), (3, 30) key = tb.RecordKey(4, col) txn.Delete(key) key = tb.RecordKey(3, col) err = tables.SetColValue(txn, key, types.NewDatum(int64(30))) c.Assert(err, IsNil) err = txn.Commit() c.Assert(err, IsNil) txn, err = s.store.Begin() c.Assert(err, IsNil) err = CompareIndexData(txn, tb, idx) c.Assert(err, NotNil) record1 = &RecordData{Handle: int64(4), Values: types.MakeDatums(int64(40))} diffMsg = newDiffRetError("index", record1, &RecordData{Handle: int64(4), Values: types.MakeDatums(nil)}) c.Assert(err.Error(), DeepEquals, diffMsg) // current index data: // index data (handle, data): (1, 10), (2, 20), (3, 30) // index col data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) err = idx.Delete(txn, types.MakeDatums(int64(40)), 4) c.Assert(err, IsNil) key = tb.RecordKey(4, col) err = tables.SetColValue(txn, key, types.NewDatum(int64(40))) c.Assert(err, IsNil) err = txn.Commit() c.Assert(err, IsNil) txn, err = s.store.Begin() c.Assert(err, IsNil) err = CompareIndexData(txn, tb, idx) c.Assert(err, NotNil) diffMsg = newDiffRetError("index", nil, record1) c.Assert(err.Error(), DeepEquals, diffMsg) }
func (s *testIndexSuite) checkPublicIndex(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, index table.Index, row []types.Datum) { t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) _, err := ctx.GetTxn(true) c.Assert(err, IsNil) i := int64(0) err = t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, row) i++ return true, nil }) c.Assert(err, IsNil) c.Assert(i, Equals, int64(1)) columnValues := make([]types.Datum, len(index.Meta().Columns)) for i, column := range index.Meta().Columns { columnValues[i] = row[column.Offset] } s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true) // Test add a new row. _, err = ctx.GetTxn(true) c.Assert(err, IsNil) newRow := types.MakeDatums(int64(11), int64(22), int64(33)) handle, err = t.AddRecord(ctx, newRow) c.Assert(err, IsNil) _, err = ctx.GetTxn(true) c.Assert(err, IsNil) rows := [][]types.Datum{row, newRow} i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { c.Assert(data, DeepEquals, rows[i]) i++ return true, nil }) c.Assert(i, Equals, int64(2)) for i, column := range index.Meta().Columns { columnValues[i] = newRow[column.Offset] } s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true) // Test update a new row. _, err = ctx.GetTxn(true) c.Assert(err, IsNil) newUpdateRow := types.MakeDatums(int64(44), int64(55), int64(66)) touched := map[int]bool{0: true, 1: true, 2: true} err = t.UpdateRecord(ctx, handle, newRow, newUpdateRow, touched) c.Assert(err, IsNil) s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false) for i, column := range index.Meta().Columns { columnValues[i] = newUpdateRow[column.Offset] } s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true) // Test remove a row. _, err = ctx.GetTxn(true) c.Assert(err, IsNil) err = t.RemoveRecord(ctx, handle, newUpdateRow) c.Assert(err, IsNil) _, err = ctx.GetTxn(true) c.Assert(err, IsNil) i = int64(0) t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { i++ return true, nil }) c.Assert(i, Equals, int64(1)) s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false) s.testGetIndex(c, t, index.Meta().Columns[0].Name.L, true) }
func (b *executorBuilder) buildIndexScan(v *plan.IndexScan) Executor { txn, err := b.ctx.GetTxn(false) if err != nil { b.err = err return nil } tbl, _ := b.is.TableByID(v.Table.ID) client := txn.GetClient() supportDesc := client.SupportRequestType(kv.ReqTypeIndex, kv.ReqSubTypeDesc) var memDB bool switch v.Fields()[0].DBName.L { case "information_schema", "performance_schema": memDB = true } if !memDB && client.SupportRequestType(kv.ReqTypeIndex, 0) { log.Debug("xapi select index") e := &XSelectIndexExec{ table: tbl, ctx: b.ctx, indexPlan: v, supportDesc: supportDesc, } where, remained := b.conditionsToPBExpr(client, v.FilterConditions, v.TableName) if where != nil { e.where = where } var ex Executor if txn.IsReadOnly() { ex = e } else { ex = b.buildUnionScanExec(e) } return b.buildFilter(ex, remained) } var idx table.Index for _, val := range tbl.Indices() { if val.Meta().Name.L == v.Index.Name.L { idx = val break } } e := &IndexScanExec{ tbl: tbl, tableAsName: v.TableAsName, idx: idx, fields: v.Fields(), ctx: b.ctx, Desc: v.Desc, valueTypes: make([]*types.FieldType, len(idx.Meta().Columns)), } for i, ic := range idx.Meta().Columns { col := tbl.Cols()[ic.Offset] e.valueTypes[i] = &col.FieldType } e.Ranges = make([]*IndexRangeExec, len(v.Ranges)) for i, val := range v.Ranges { e.Ranges[i] = b.buildIndexRange(e, val) } x := b.buildFilter(e, v.FilterConditions) if v.Desc { x = &ReverseExec{Src: x} } return x }