func (s *testIndexSuite) TestCombineIndexSeek(c *C) { defer testleak.AfterTest(c)() tblInfo := &model.TableInfo{ ID: 1, Indices: []*model.IndexInfo{ { ID: 2, Name: model.NewCIStr("test"), Columns: []*model.IndexColumn{ {}, {}, }, }, }, } index := tables.NewIndex(tblInfo, tblInfo.Indices[0]) txn, err := s.s.Begin() c.Assert(err, IsNil) values := types.MakeDatums("abc", "def") _, err = index.Create(txn, values, 1) c.Assert(err, IsNil) index2 := tables.NewIndex(tblInfo, tblInfo.Indices[0]) iter, hit, err := index2.Seek(txn, types.MakeDatums("abc", nil)) c.Assert(err, IsNil) defer iter.Close() c.Assert(hit, IsFalse) _, h, err := iter.Next() c.Assert(err, IsNil) c.Assert(h, Equals, int64(1)) }
func checkRecordAndIndex(txn kv.Transaction, t table.Table, idx *table.IndexedColumn) error { cols := make([]*table.Column, len(idx.Columns)) for i, col := range idx.Columns { cols[i] = t.Cols()[col.Offset] } startKey := t.RecordKey(0, nil) kvIndex := tables.NewIndex(t.IndexPrefix(), idx.Name.L, idx.ID, idx.Unique) filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) { isExist, h2, err := kvIndex.Exist(txn, vals1, h1) if terror.ErrorEqual(err, kv.ErrKeyExists) { record1 := &RecordData{Handle: h1, Values: vals1} record2 := &RecordData{Handle: h2, Values: vals1} return false, errDateNotEqual.Gen("index:%v != record:%v", record2, record1) } if err != nil { return false, errors.Trace(err) } if !isExist { record := &RecordData{Handle: h1, Values: vals1} return false, errDateNotEqual.Gen("index:%v != record:%v", nil, record) } return true, nil } err := iterRecords(txn, t, startKey, cols, filterFunc) if err != nil { return errors.Trace(err) } return nil }
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error { kvX := tables.NewIndex(t.Meta(), indexInfo) for _, handle := range handles { log.Debug("[ddl] building index...", handle) err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { if err := d.isReorgRunnable(txn); err != nil { return errors.Trace(err) } // first check row exists exist, err := checkRowExist(txn, t, handle) if err != nil { return errors.Trace(err) } else if !exist { // row doesn't exist, skip it. return nil } var vals []types.Datum vals, err = fetchRowColVals(txn, t, handle, indexInfo) if err != nil { return errors.Trace(err) } exist, _, err = kvX.Exist(txn, vals, handle) if err != nil { return errors.Trace(err) } else if exist { // index already exists, skip it. return nil } err = lockRow(txn, t, handle) if err != nil { return errors.Trace(err) } // create the index. err = kvX.Create(txn, vals, handle) if err != nil { return errors.Trace(err) } // update reorg next handle return errors.Trace(reorgInfo.UpdateHandle(txn, handle)) }) if err != nil { return errors.Trace(err) } } return nil }
func (s *testIndexSuite) TestCombineIndexSeek(c *C) { defer testleak.AfterTest(c)() index := tables.NewIndex([]byte("i"), "test", 1, false) txn, err := s.s.Begin() c.Assert(err, IsNil) values := types.MakeDatums("abc", "def") err = index.Create(txn, values, 1) c.Assert(err, IsNil) index2 := tables.NewIndex([]byte("i"), "test", 1, false) iter, hit, err := index2.Seek(txn, types.MakeDatums("abc", nil)) c.Assert(err, IsNil) defer iter.Close() c.Assert(hit, IsFalse) _, h, err := iter.Next() c.Assert(err, IsNil) c.Assert(h, Equals, int64(1)) }
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error { kvX := tables.NewIndex(t.Meta(), indexInfo) for _, handle := range handles { log.Debug("[ddl] building index...", handle) err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { if err := d.isReorgRunnable(txn); err != nil { return errors.Trace(err) } vals, err1 := fetchRowColVals(txn, t, handle, indexInfo) if terror.ErrorEqual(err1, kv.ErrNotExist) { // row doesn't exist, skip it. return nil } if err1 != nil { return errors.Trace(err1) } exist, _, err1 := kvX.Exist(txn, vals, handle) if err1 != nil { return errors.Trace(err1) } else if exist { // index already exists, skip it. return nil } rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle) err1 = txn.LockKeys(rowKey) if err1 != nil { return errors.Trace(err1) } // create the index. err1 = kvX.Create(txn, vals, handle) if err1 != nil { return errors.Trace(err1) } // update reorg next handle return errors.Trace(reorgInfo.UpdateHandle(txn, handle)) }) if err != nil { return errors.Trace(err) } } return nil }
func checkIndexAndRecord(txn kv.Transaction, t table.Table, idx *table.IndexedColumn) error { kvIndex := tables.NewIndex(t.IndexPrefix(), idx.Name.L, idx.ID, idx.Unique) it, err := kvIndex.SeekFirst(txn) if err != nil { return errors.Trace(err) } defer it.Close() cols := make([]*table.Column, len(idx.Columns)) for i, col := range idx.Columns { cols[i] = t.Cols()[col.Offset] } for { vals1, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } else if err != nil { return errors.Trace(err) } vals2, err := rowWithCols(txn, t, h, cols) if terror.ErrorEqual(err, kv.ErrNotExist) { record := &RecordData{Handle: h, Values: vals1} err = errDateNotEqual.Gen("index:%v != record:%v", record, nil) } if err != nil { return errors.Trace(err) } if !reflect.DeepEqual(vals1, vals2) { record1 := &RecordData{Handle: h, Values: vals1} record2 := &RecordData{Handle: h, Values: vals2} return errDateNotEqual.Gen("index:%v != record:%v", record1, record2) } } return nil }
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error { kvIdx := tables.NewIndex(t.Meta(), indexInfo) for len(handles) > 0 { endIdx := int(math.Min(float64(defaultSmallBatchCnt), float64(len(handles)))) err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { if err1 := d.isReorgRunnable(txn, ddlJobFlag); err1 != nil { return errors.Trace(err1) } nextHandle, err1 := d.backfillIndexInTxn(t, kvIdx, handles[:endIdx], txn) if err1 != nil { return errors.Trace(err1) } // Update reorg next handle. return errors.Trace(reorgInfo.UpdateHandle(txn, nextHandle)) }) if err != nil { return errors.Trace(err) } handles = handles[endIdx:] } return nil }
func (s *testDBSuite) testDropIndex(c *C) { done := make(chan struct{}, 1) s.mustExec(c, "delete from t1") num := 100 // add some rows for i := 0; i < num; i++ { s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } t := s.testGetTable(c, "t1") var c3idx table.Index for _, tidx := range t.Indices() { if tidx.Meta().Name.L == "c3_index" { c3idx = tidx break } } c.Assert(c3idx, NotNil) go func() { s.mustExec(c, "drop index c3_index on t1") done <- struct{}{} }() ticker := time.NewTicker(s.lease / 2) defer ticker.Stop() LOOP: for { select { case <-done: break LOOP case <-ticker.C: step := 10 // delete some rows, and add some data for i := num; i < num+step; i++ { n := rand.Intn(num) s.mustExec(c, "update t1 set c2 = 1 where c1 = ?", n) s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } num += step } } rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 0") ay := dumpRows(c, rows) c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsFalse) // check in index, must no index in kv ctx := s.s.(context.Context) handles := make(map[int64]struct{}) t = s.testGetTable(c, "t1") var nidx table.Index for _, tidx := range t.Indices() { if tidx.Meta().Name.L == "c3_index" { nidx = tidx break } } // Make sure there is no index with name c3_index c.Assert(nidx, IsNil) idx := tables.NewIndex(t.Meta(), c3idx.Meta()) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) defer ctx.RollbackTxn() it, err := idx.SeekFirst(txn) c.Assert(err, IsNil) defer it.Close() for { _, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } c.Assert(err, IsNil) handles[h] = struct{}{} } c.Assert(handles, HasLen, 0) }
func (s *testSuite) TestScan(c *C) { defer testleak.AfterTest(c)() alloc := autoid.NewAllocator(s.store, s.dbInfo.ID) tb, err := tables.TableFromMeta(alloc, s.tbInfo) c.Assert(err, IsNil) indices := tb.Indices() _, err = tb.AddRecord(s.ctx, types.MakeDatums(10, 11)) c.Assert(err, IsNil) s.ctx.CommitTxn() record1 := &RecordData{Handle: int64(1), Values: types.MakeDatums(int64(10), int64(11))} record2 := &RecordData{Handle: int64(2), Values: types.MakeDatums(int64(20), int64(21))} ver, err := s.store.CurrentVersion() c.Assert(err, IsNil) records, _, err := ScanSnapshotTableRecord(s.store, ver, tb, int64(1), 1) c.Assert(err, IsNil) c.Assert(records, DeepEquals, []*RecordData{record1}) _, err = tb.AddRecord(s.ctx, record2.Values) c.Assert(err, IsNil) s.ctx.CommitTxn() txn, err := s.store.Begin() c.Assert(err, IsNil) records, nextHandle, err := ScanTableRecord(txn, tb, int64(1), 1) c.Assert(err, IsNil) c.Assert(records, DeepEquals, []*RecordData{record1}) records, nextHandle, err = ScanTableRecord(txn, tb, nextHandle, 1) c.Assert(err, IsNil) c.Assert(records, DeepEquals, []*RecordData{record2}) startHandle := nextHandle records, nextHandle, err = ScanTableRecord(txn, tb, startHandle, 1) c.Assert(err, IsNil) c.Assert(records, IsNil) c.Assert(nextHandle, Equals, startHandle) idxRow1 := &RecordData{Handle: int64(1), Values: types.MakeDatums(int64(10))} idxRow2 := &RecordData{Handle: int64(2), Values: types.MakeDatums(int64(20))} kvIndex := tables.NewIndex(tb.Meta(), indices[0].Meta()) idxRows, nextVals, err := ScanIndexData(txn, kvIndex, idxRow1.Values, 2) c.Assert(err, IsNil) c.Assert(idxRows, DeepEquals, []*RecordData{idxRow1, idxRow2}) idxRows, nextVals, err = ScanIndexData(txn, kvIndex, idxRow1.Values, 1) c.Assert(err, IsNil) c.Assert(idxRows, DeepEquals, []*RecordData{idxRow1}) idxRows, nextVals, err = ScanIndexData(txn, kvIndex, nextVals, 1) c.Assert(err, IsNil) c.Assert(idxRows, DeepEquals, []*RecordData{idxRow2}) idxRows, nextVals, err = ScanIndexData(txn, kvIndex, nextVals, 1) c.Assert(idxRows, IsNil) c.Assert(nextVals, DeepEquals, types.MakeDatums(nil)) c.Assert(err, IsNil) s.testTableData(c, tb, []*RecordData{record1, record2}) s.testIndex(c, tb, tb.Indices()[0]) err = tb.RemoveRecord(s.ctx, 1, record1.Values) c.Assert(err, IsNil) err = tb.RemoveRecord(s.ctx, 2, record2.Values) c.Assert(err, IsNil) }
func (s *testIndexSuite) TestDropIndex(c *C) { defer testleak.AfterTest(c)() d := newDDL(s.store, nil, nil, testLease) tblInfo := testTableInfo(c, d, "t", 3) ctx := testNewContext(c, d) _, err := ctx.GetTxn(true) c.Assert(err, IsNil) testCreateTable(c, ctx, d, s.dbInfo, tblInfo) t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) row := types.MakeDatums(int64(1), int64(2), int64(3)) handle, err := t.AddRecord(ctx, row) c.Assert(err, IsNil) err = ctx.CommitTxn() c.Assert(err, IsNil) job := testCreateIndex(c, ctx, s.d, s.dbInfo, tblInfo, true, "c1_uni", "c1") testCheckJobDone(c, d, job, true) err = ctx.CommitTxn() c.Assert(err, IsNil) checkOK := false oldIndexCol := tables.NewIndex(tblInfo, &model.IndexInfo{}) tc := &testDDLCallback{} tc.onJobUpdated = func(job *model.Job) { if checkOK { return } t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID) index := getIndex(t, "c1") if index == nil { s.checkAddOrDropIndex(c, model.StateNone, d, tblInfo, handle, oldIndexCol, row, true) checkOK = true return } s.checkAddOrDropIndex(c, index.Meta().State, d, tblInfo, handle, index, row, true) oldIndexCol = index } d.m.Lock() d.hook = tc d.m.Unlock() // Use local ddl for callback test. s.d.close() d.close() d.start() job = testDropIndex(c, ctx, d, s.dbInfo, tblInfo, "c1_uni") testCheckJobDone(c, d, job, false) _, err = ctx.GetTxn(true) c.Assert(err, IsNil) job = testDropTable(c, ctx, d, s.dbInfo, tblInfo) testCheckJobDone(c, d, job, false) err = ctx.CommitTxn() c.Assert(err, IsNil) d.close() s.d.start() }
func (s *testDBSuite) testAddIndex(c *C) { done := make(chan struct{}, 1) num := 100 // first add some rows for i := 0; i < num; i++ { s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } go func() { s.mustExec(c, "create index c3_index on t1 (c3)") done <- struct{}{} }() deletedKeys := make(map[int]struct{}) ticker := time.NewTicker(s.lease / 2) defer ticker.Stop() LOOP: for { select { case <-done: break LOOP case <-ticker.C: step := 10 // delete some rows, and add some data for i := num; i < num+step; i++ { n := rand.Intn(num) deletedKeys[n] = struct{}{} s.mustExec(c, "delete from t1 where c1 = ?", n) s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i) } num += step } } // get exists keys keys := make([]int, 0, num) for i := 0; i < num; i++ { if _, ok := deletedKeys[i]; ok { continue } keys = append(keys, i) } // test index key for _, key := range keys { rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key) matchRows(c, rows, [][]interface{}{{key}}) } // test delete key not in index for key := range deletedKeys { rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key) matchRows(c, rows, nil) } // test index range for i := 0; i < 100; i++ { index := rand.Intn(len(keys) - 3) rows := s.mustQuery(c, "select c1 from t1 where c3 >= ? limit 3", keys[index]) matchRows(c, rows, [][]interface{}{{keys[index]}, {keys[index+1]}, {keys[index+2]}}) } rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 100") ay := dumpRows(c, rows) c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsTrue) // get all row handles ctx := s.s.(context.Context) t := s.testGetTable(c, "t1") handles := make(map[int64]struct{}) err := t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) { handles[h] = struct{}{} return true, nil }) c.Assert(err, IsNil) // check in index var nidx *table.IndexedColumn for _, tidx := range t.Indices() { if tidx.Name.L == "c3_index" { nidx = tidx break } } // Make sure there is index with name c3_index c.Assert(nidx, NotNil) c.Assert(nidx.ID, Greater, int64(0)) idx := tables.NewIndex(t.IndexPrefix(), "c3_index", nidx.ID, false) txn, err := ctx.GetTxn(true) c.Assert(err, IsNil) defer ctx.RollbackTxn() it, err := idx.SeekFirst(txn) c.Assert(err, IsNil) defer it.Close() for { _, h, err := it.Next() if terror.ErrorEqual(err, io.EOF) { break } c.Assert(err, IsNil) _, ok := handles[h] c.Assert(ok, IsTrue) delete(handles, h) } c.Assert(handles, HasLen, 0) }
func (s *testIndexSuite) TestIndex(c *C) { defer testleak.AfterTest(c)() tblInfo := &model.TableInfo{ ID: 1, Indices: []*model.IndexInfo{ { ID: 2, Name: model.NewCIStr("test"), Columns: []*model.IndexColumn{ {}, {}, }, }, }, } index := tables.NewIndex(tblInfo, tblInfo.Indices[0]) // Test ununiq index. txn, err := s.s.Begin() c.Assert(err, IsNil) values := types.MakeDatums(1, 2) _, err = index.Create(txn, values, 1) c.Assert(err, IsNil) it, err := index.SeekFirst(txn) c.Assert(err, IsNil) getValues, h, err := it.Next() c.Assert(err, IsNil) c.Assert(getValues, HasLen, 2) c.Assert(getValues[0].GetInt64(), Equals, int64(1)) c.Assert(getValues[1].GetInt64(), Equals, int64(2)) c.Assert(h, Equals, int64(1)) it.Close() exist, _, err := index.Exist(txn, values, 100) c.Assert(err, IsNil) c.Assert(exist, IsFalse) exist, _, err = index.Exist(txn, values, 1) c.Assert(err, IsNil) c.Assert(exist, IsTrue) err = index.Delete(txn, values, 1) c.Assert(err, IsNil) it, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() _, err = index.Create(txn, values, 0) c.Assert(err, IsNil) _, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, hit, err := index.Seek(txn, values) c.Assert(err, IsNil) c.Assert(hit, IsTrue) err = index.Drop(txn) c.Assert(err, IsNil) it, hit, err = index.Seek(txn, values) c.Assert(err, IsNil) c.Assert(hit, IsFalse) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() it, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() err = txn.Commit() c.Assert(err, IsNil) tblInfo = &model.TableInfo{ ID: 2, Indices: []*model.IndexInfo{ { ID: 3, Name: model.NewCIStr("test"), Unique: true, Columns: []*model.IndexColumn{ {}, {}, }, }, }, } index = tables.NewIndex(tblInfo, tblInfo.Indices[0]) // Test uniq index. txn, err = s.s.Begin() c.Assert(err, IsNil) _, err = index.Create(txn, values, 1) c.Assert(err, IsNil) _, err = index.Create(txn, values, 2) c.Assert(err, NotNil) exist, h, err = index.Exist(txn, values, 1) c.Assert(err, IsNil) c.Assert(h, Equals, int64(1)) c.Assert(exist, IsTrue) exist, h, err = index.Exist(txn, values, 2) c.Assert(err, NotNil) c.Assert(h, Equals, int64(1)) c.Assert(exist, IsTrue) err = txn.Commit() c.Assert(err, IsNil) _, err = index.FetchValues(make([]types.Datum, 0)) c.Assert(err, NotNil) }
func (s *testIndexSuite) TestIndex(c *C) { defer testleak.AfterTest(c)() index := tables.NewIndex([]byte("i"), "test", 0, false) // Test ununiq index. txn, err := s.s.Begin() c.Assert(err, IsNil) values := types.MakeDatums(1, 2) err = index.Create(txn, values, 1) c.Assert(err, IsNil) it, err := index.SeekFirst(txn) c.Assert(err, IsNil) getValues, h, err := it.Next() c.Assert(err, IsNil) c.Assert(getValues, HasLen, 2) c.Assert(getValues[0].GetInt64(), Equals, int64(1)) c.Assert(getValues[1].GetInt64(), Equals, int64(2)) c.Assert(h, Equals, int64(1)) it.Close() exist, _, err := index.Exist(txn, values, 100) c.Assert(err, IsNil) c.Assert(exist, IsFalse) exist, _, err = index.Exist(txn, values, 1) c.Assert(err, IsNil) c.Assert(exist, IsTrue) err = index.Delete(txn, values, 1) c.Assert(err, IsNil) it, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() err = index.Create(txn, values, 0) c.Assert(err, IsNil) _, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, hit, err := index.Seek(txn, values) c.Assert(err, IsNil) c.Assert(hit, IsTrue) err = index.Drop(txn) c.Assert(err, IsNil) it, hit, err = index.Seek(txn, values) c.Assert(err, IsNil) c.Assert(hit, IsFalse) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() it, err = index.SeekFirst(txn) c.Assert(err, IsNil) _, _, err = it.Next() c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue) it.Close() err = txn.Commit() c.Assert(err, IsNil) index = tables.NewIndex([]byte("j"), "test", 1, true) // Test uniq index. txn, err = s.s.Begin() c.Assert(err, IsNil) err = index.Create(txn, values, 1) c.Assert(err, IsNil) err = index.Create(txn, values, 2) c.Assert(err, NotNil) exist, h, err = index.Exist(txn, values, 1) c.Assert(err, IsNil) c.Assert(h, Equals, int64(1)) c.Assert(exist, IsTrue) exist, h, err = index.Exist(txn, values, 2) c.Assert(err, NotNil) c.Assert(h, Equals, int64(1)) c.Assert(exist, IsTrue) err = txn.Commit() c.Assert(err, IsNil) }