func (s *testIndexSuite) TestCombineIndexSeek(c *C) {
defer testleak.AfterTest(c)()
tblInfo := &model.TableInfo{
ID: 1,
Indices: []*model.IndexInfo{
{
ID: 2,
Name: model.NewCIStr("test"),
Columns: []*model.IndexColumn{
{},
{},
},
},
},
}
index := tables.NewIndex(tblInfo, tblInfo.Indices[0])
txn, err := s.s.Begin()
c.Assert(err, IsNil)
values := types.MakeDatums("abc", "def")
_, err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
index2 := tables.NewIndex(tblInfo, tblInfo.Indices[0])
iter, hit, err := index2.Seek(txn, types.MakeDatums("abc", nil))
c.Assert(err, IsNil)
defer iter.Close()
c.Assert(hit, IsFalse)
_, h, err := iter.Next()
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(1))
}
func checkRecordAndIndex(txn kv.Transaction, t table.Table, idx *table.IndexedColumn) error {
cols := make([]*table.Column, len(idx.Columns))
for i, col := range idx.Columns {
cols[i] = t.Cols()[col.Offset]
}
startKey := t.RecordKey(0, nil)
kvIndex := tables.NewIndex(t.IndexPrefix(), idx.Name.L, idx.ID, idx.Unique)
filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) {
isExist, h2, err := kvIndex.Exist(txn, vals1, h1)
if terror.ErrorEqual(err, kv.ErrKeyExists) {
record1 := &RecordData{Handle: h1, Values: vals1}
record2 := &RecordData{Handle: h2, Values: vals1}
return false, errDateNotEqual.Gen("index:%v != record:%v", record2, record1)
}
if err != nil {
return false, errors.Trace(err)
}
if !isExist {
record := &RecordData{Handle: h1, Values: vals1}
return false, errDateNotEqual.Gen("index:%v != record:%v", nil, record)
}
return true, nil
}
err := iterRecords(txn, t, startKey, cols, filterFunc)
if err != nil {
return errors.Trace(err)
}
return nil
}
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error {
kvX := tables.NewIndex(t.Meta(), indexInfo)
for _, handle := range handles {
log.Debug("[ddl] building index...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
// first check row exists
exist, err := checkRowExist(txn, t, handle)
if err != nil {
return errors.Trace(err)
} else if !exist {
// row doesn't exist, skip it.
return nil
}
var vals []types.Datum
vals, err = fetchRowColVals(txn, t, handle, indexInfo)
if err != nil {
return errors.Trace(err)
}
exist, _, err = kvX.Exist(txn, vals, handle)
if err != nil {
return errors.Trace(err)
} else if exist {
// index already exists, skip it.
return nil
}
err = lockRow(txn, t, handle)
if err != nil {
return errors.Trace(err)
}
// create the index.
err = kvX.Create(txn, vals, handle)
if err != nil {
return errors.Trace(err)
}
// update reorg next handle
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func (s *testIndexSuite) TestCombineIndexSeek(c *C) {
defer testleak.AfterTest(c)()
index := tables.NewIndex([]byte("i"), "test", 1, false)
txn, err := s.s.Begin()
c.Assert(err, IsNil)
values := types.MakeDatums("abc", "def")
err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
index2 := tables.NewIndex([]byte("i"), "test", 1, false)
iter, hit, err := index2.Seek(txn, types.MakeDatums("abc", nil))
c.Assert(err, IsNil)
defer iter.Close()
c.Assert(hit, IsFalse)
_, h, err := iter.Next()
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(1))
}
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error {
kvX := tables.NewIndex(t.Meta(), indexInfo)
for _, handle := range handles {
log.Debug("[ddl] building index...", handle)
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err := d.isReorgRunnable(txn); err != nil {
return errors.Trace(err)
}
vals, err1 := fetchRowColVals(txn, t, handle, indexInfo)
if terror.ErrorEqual(err1, kv.ErrNotExist) {
// row doesn't exist, skip it.
return nil
}
if err1 != nil {
return errors.Trace(err1)
}
exist, _, err1 := kvX.Exist(txn, vals, handle)
if err1 != nil {
return errors.Trace(err1)
} else if exist {
// index already exists, skip it.
return nil
}
rowKey := tablecodec.EncodeRecordKey(t.RecordPrefix(), handle)
err1 = txn.LockKeys(rowKey)
if err1 != nil {
return errors.Trace(err1)
}
// create the index.
err1 = kvX.Create(txn, vals, handle)
if err1 != nil {
return errors.Trace(err1)
}
// update reorg next handle
return errors.Trace(reorgInfo.UpdateHandle(txn, handle))
})
if err != nil {
return errors.Trace(err)
}
}
return nil
}
func checkIndexAndRecord(txn kv.Transaction, t table.Table, idx *table.IndexedColumn) error {
kvIndex := tables.NewIndex(t.IndexPrefix(), idx.Name.L, idx.ID, idx.Unique)
it, err := kvIndex.SeekFirst(txn)
if err != nil {
return errors.Trace(err)
}
defer it.Close()
cols := make([]*table.Column, len(idx.Columns))
for i, col := range idx.Columns {
cols[i] = t.Cols()[col.Offset]
}
for {
vals1, h, err := it.Next()
if terror.ErrorEqual(err, io.EOF) {
break
} else if err != nil {
return errors.Trace(err)
}
vals2, err := rowWithCols(txn, t, h, cols)
if terror.ErrorEqual(err, kv.ErrNotExist) {
record := &RecordData{Handle: h, Values: vals1}
err = errDateNotEqual.Gen("index:%v != record:%v", record, nil)
}
if err != nil {
return errors.Trace(err)
}
if !reflect.DeepEqual(vals1, vals2) {
record1 := &RecordData{Handle: h, Values: vals1}
record2 := &RecordData{Handle: h, Values: vals2}
return errDateNotEqual.Gen("index:%v != record:%v", record1, record2)
}
}
return nil
}
func (d *ddl) backfillTableIndex(t table.Table, indexInfo *model.IndexInfo, handles []int64, reorgInfo *reorgInfo) error {
kvIdx := tables.NewIndex(t.Meta(), indexInfo)
for len(handles) > 0 {
endIdx := int(math.Min(float64(defaultSmallBatchCnt), float64(len(handles))))
err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error {
if err1 := d.isReorgRunnable(txn, ddlJobFlag); err1 != nil {
return errors.Trace(err1)
}
nextHandle, err1 := d.backfillIndexInTxn(t, kvIdx, handles[:endIdx], txn)
if err1 != nil {
return errors.Trace(err1)
}
// Update reorg next handle.
return errors.Trace(reorgInfo.UpdateHandle(txn, nextHandle))
})
if err != nil {
return errors.Trace(err)
}
handles = handles[endIdx:]
}
return nil
}
func (s *testDBSuite) testDropIndex(c *C) {
done := make(chan struct{}, 1)
s.mustExec(c, "delete from t1")
num := 100
// add some rows
for i := 0; i < num; i++ {
s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
}
t := s.testGetTable(c, "t1")
var c3idx table.Index
for _, tidx := range t.Indices() {
if tidx.Meta().Name.L == "c3_index" {
c3idx = tidx
break
}
}
c.Assert(c3idx, NotNil)
go func() {
s.mustExec(c, "drop index c3_index on t1")
done <- struct{}{}
}()
ticker := time.NewTicker(s.lease / 2)
defer ticker.Stop()
LOOP:
for {
select {
case <-done:
break LOOP
case <-ticker.C:
step := 10
// delete some rows, and add some data
for i := num; i < num+step; i++ {
n := rand.Intn(num)
s.mustExec(c, "update t1 set c2 = 1 where c1 = ?", n)
s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
}
num += step
}
}
rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 0")
ay := dumpRows(c, rows)
c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsFalse)
// check in index, must no index in kv
ctx := s.s.(context.Context)
handles := make(map[int64]struct{})
t = s.testGetTable(c, "t1")
var nidx table.Index
for _, tidx := range t.Indices() {
if tidx.Meta().Name.L == "c3_index" {
nidx = tidx
break
}
}
// Make sure there is no index with name c3_index
c.Assert(nidx, IsNil)
idx := tables.NewIndex(t.Meta(), c3idx.Meta())
txn, err := ctx.GetTxn(true)
c.Assert(err, IsNil)
defer ctx.RollbackTxn()
it, err := idx.SeekFirst(txn)
c.Assert(err, IsNil)
defer it.Close()
for {
_, h, err := it.Next()
if terror.ErrorEqual(err, io.EOF) {
break
}
c.Assert(err, IsNil)
handles[h] = struct{}{}
}
c.Assert(handles, HasLen, 0)
}
func (s *testSuite) TestScan(c *C) {
defer testleak.AfterTest(c)()
alloc := autoid.NewAllocator(s.store, s.dbInfo.ID)
tb, err := tables.TableFromMeta(alloc, s.tbInfo)
c.Assert(err, IsNil)
indices := tb.Indices()
_, err = tb.AddRecord(s.ctx, types.MakeDatums(10, 11))
c.Assert(err, IsNil)
s.ctx.CommitTxn()
record1 := &RecordData{Handle: int64(1), Values: types.MakeDatums(int64(10), int64(11))}
record2 := &RecordData{Handle: int64(2), Values: types.MakeDatums(int64(20), int64(21))}
ver, err := s.store.CurrentVersion()
c.Assert(err, IsNil)
records, _, err := ScanSnapshotTableRecord(s.store, ver, tb, int64(1), 1)
c.Assert(err, IsNil)
c.Assert(records, DeepEquals, []*RecordData{record1})
_, err = tb.AddRecord(s.ctx, record2.Values)
c.Assert(err, IsNil)
s.ctx.CommitTxn()
txn, err := s.store.Begin()
c.Assert(err, IsNil)
records, nextHandle, err := ScanTableRecord(txn, tb, int64(1), 1)
c.Assert(err, IsNil)
c.Assert(records, DeepEquals, []*RecordData{record1})
records, nextHandle, err = ScanTableRecord(txn, tb, nextHandle, 1)
c.Assert(err, IsNil)
c.Assert(records, DeepEquals, []*RecordData{record2})
startHandle := nextHandle
records, nextHandle, err = ScanTableRecord(txn, tb, startHandle, 1)
c.Assert(err, IsNil)
c.Assert(records, IsNil)
c.Assert(nextHandle, Equals, startHandle)
idxRow1 := &RecordData{Handle: int64(1), Values: types.MakeDatums(int64(10))}
idxRow2 := &RecordData{Handle: int64(2), Values: types.MakeDatums(int64(20))}
kvIndex := tables.NewIndex(tb.Meta(), indices[0].Meta())
idxRows, nextVals, err := ScanIndexData(txn, kvIndex, idxRow1.Values, 2)
c.Assert(err, IsNil)
c.Assert(idxRows, DeepEquals, []*RecordData{idxRow1, idxRow2})
idxRows, nextVals, err = ScanIndexData(txn, kvIndex, idxRow1.Values, 1)
c.Assert(err, IsNil)
c.Assert(idxRows, DeepEquals, []*RecordData{idxRow1})
idxRows, nextVals, err = ScanIndexData(txn, kvIndex, nextVals, 1)
c.Assert(err, IsNil)
c.Assert(idxRows, DeepEquals, []*RecordData{idxRow2})
idxRows, nextVals, err = ScanIndexData(txn, kvIndex, nextVals, 1)
c.Assert(idxRows, IsNil)
c.Assert(nextVals, DeepEquals, types.MakeDatums(nil))
c.Assert(err, IsNil)
s.testTableData(c, tb, []*RecordData{record1, record2})
s.testIndex(c, tb, tb.Indices()[0])
err = tb.RemoveRecord(s.ctx, 1, record1.Values)
c.Assert(err, IsNil)
err = tb.RemoveRecord(s.ctx, 2, record2.Values)
c.Assert(err, IsNil)
}
func (s *testIndexSuite) TestDropIndex(c *C) {
defer testleak.AfterTest(c)()
d := newDDL(s.store, nil, nil, testLease)
tblInfo := testTableInfo(c, d, "t", 3)
ctx := testNewContext(c, d)
_, err := ctx.GetTxn(true)
c.Assert(err, IsNil)
testCreateTable(c, ctx, d, s.dbInfo, tblInfo)
t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID)
row := types.MakeDatums(int64(1), int64(2), int64(3))
handle, err := t.AddRecord(ctx, row)
c.Assert(err, IsNil)
err = ctx.CommitTxn()
c.Assert(err, IsNil)
job := testCreateIndex(c, ctx, s.d, s.dbInfo, tblInfo, true, "c1_uni", "c1")
testCheckJobDone(c, d, job, true)
err = ctx.CommitTxn()
c.Assert(err, IsNil)
checkOK := false
oldIndexCol := tables.NewIndex(tblInfo, &model.IndexInfo{})
tc := &testDDLCallback{}
tc.onJobUpdated = func(job *model.Job) {
if checkOK {
return
}
t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID)
index := getIndex(t, "c1")
if index == nil {
s.checkAddOrDropIndex(c, model.StateNone, d, tblInfo, handle, oldIndexCol, row, true)
checkOK = true
return
}
s.checkAddOrDropIndex(c, index.Meta().State, d, tblInfo, handle, index, row, true)
oldIndexCol = index
}
d.m.Lock()
d.hook = tc
d.m.Unlock()
// Use local ddl for callback test.
s.d.close()
d.close()
d.start()
job = testDropIndex(c, ctx, d, s.dbInfo, tblInfo, "c1_uni")
testCheckJobDone(c, d, job, false)
_, err = ctx.GetTxn(true)
c.Assert(err, IsNil)
job = testDropTable(c, ctx, d, s.dbInfo, tblInfo)
testCheckJobDone(c, d, job, false)
err = ctx.CommitTxn()
c.Assert(err, IsNil)
d.close()
s.d.start()
}
func (s *testDBSuite) testAddIndex(c *C) {
done := make(chan struct{}, 1)
num := 100
// first add some rows
for i := 0; i < num; i++ {
s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
}
go func() {
s.mustExec(c, "create index c3_index on t1 (c3)")
done <- struct{}{}
}()
deletedKeys := make(map[int]struct{})
ticker := time.NewTicker(s.lease / 2)
defer ticker.Stop()
LOOP:
for {
select {
case <-done:
break LOOP
case <-ticker.C:
step := 10
// delete some rows, and add some data
for i := num; i < num+step; i++ {
n := rand.Intn(num)
deletedKeys[n] = struct{}{}
s.mustExec(c, "delete from t1 where c1 = ?", n)
s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
}
num += step
}
}
// get exists keys
keys := make([]int, 0, num)
for i := 0; i < num; i++ {
if _, ok := deletedKeys[i]; ok {
continue
}
keys = append(keys, i)
}
// test index key
for _, key := range keys {
rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key)
matchRows(c, rows, [][]interface{}{{key}})
}
// test delete key not in index
for key := range deletedKeys {
rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key)
matchRows(c, rows, nil)
}
// test index range
for i := 0; i < 100; i++ {
index := rand.Intn(len(keys) - 3)
rows := s.mustQuery(c, "select c1 from t1 where c3 >= ? limit 3", keys[index])
matchRows(c, rows, [][]interface{}{{keys[index]}, {keys[index+1]}, {keys[index+2]}})
}
rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 100")
ay := dumpRows(c, rows)
c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsTrue)
// get all row handles
ctx := s.s.(context.Context)
t := s.testGetTable(c, "t1")
handles := make(map[int64]struct{})
err := t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) {
handles[h] = struct{}{}
return true, nil
})
c.Assert(err, IsNil)
// check in index
var nidx *table.IndexedColumn
for _, tidx := range t.Indices() {
if tidx.Name.L == "c3_index" {
nidx = tidx
break
}
}
// Make sure there is index with name c3_index
c.Assert(nidx, NotNil)
c.Assert(nidx.ID, Greater, int64(0))
idx := tables.NewIndex(t.IndexPrefix(), "c3_index", nidx.ID, false)
txn, err := ctx.GetTxn(true)
c.Assert(err, IsNil)
defer ctx.RollbackTxn()
it, err := idx.SeekFirst(txn)
c.Assert(err, IsNil)
defer it.Close()
for {
_, h, err := it.Next()
if terror.ErrorEqual(err, io.EOF) {
break
}
c.Assert(err, IsNil)
_, ok := handles[h]
c.Assert(ok, IsTrue)
delete(handles, h)
}
c.Assert(handles, HasLen, 0)
}
func (s *testIndexSuite) TestIndex(c *C) {
defer testleak.AfterTest(c)()
tblInfo := &model.TableInfo{
ID: 1,
Indices: []*model.IndexInfo{
{
ID: 2,
Name: model.NewCIStr("test"),
Columns: []*model.IndexColumn{
{},
{},
},
},
},
}
index := tables.NewIndex(tblInfo, tblInfo.Indices[0])
// Test ununiq index.
txn, err := s.s.Begin()
c.Assert(err, IsNil)
values := types.MakeDatums(1, 2)
_, err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
it, err := index.SeekFirst(txn)
c.Assert(err, IsNil)
getValues, h, err := it.Next()
c.Assert(err, IsNil)
c.Assert(getValues, HasLen, 2)
c.Assert(getValues[0].GetInt64(), Equals, int64(1))
c.Assert(getValues[1].GetInt64(), Equals, int64(2))
c.Assert(h, Equals, int64(1))
it.Close()
exist, _, err := index.Exist(txn, values, 100)
c.Assert(err, IsNil)
c.Assert(exist, IsFalse)
exist, _, err = index.Exist(txn, values, 1)
c.Assert(err, IsNil)
c.Assert(exist, IsTrue)
err = index.Delete(txn, values, 1)
c.Assert(err, IsNil)
it, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
_, err = index.Create(txn, values, 0)
c.Assert(err, IsNil)
_, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, hit, err := index.Seek(txn, values)
c.Assert(err, IsNil)
c.Assert(hit, IsTrue)
err = index.Drop(txn)
c.Assert(err, IsNil)
it, hit, err = index.Seek(txn, values)
c.Assert(err, IsNil)
c.Assert(hit, IsFalse)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
it, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
err = txn.Commit()
c.Assert(err, IsNil)
tblInfo = &model.TableInfo{
ID: 2,
Indices: []*model.IndexInfo{
{
ID: 3,
Name: model.NewCIStr("test"),
Unique: true,
Columns: []*model.IndexColumn{
{},
{},
},
},
},
}
index = tables.NewIndex(tblInfo, tblInfo.Indices[0])
// Test uniq index.
txn, err = s.s.Begin()
c.Assert(err, IsNil)
_, err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
_, err = index.Create(txn, values, 2)
c.Assert(err, NotNil)
exist, h, err = index.Exist(txn, values, 1)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(1))
c.Assert(exist, IsTrue)
exist, h, err = index.Exist(txn, values, 2)
c.Assert(err, NotNil)
c.Assert(h, Equals, int64(1))
c.Assert(exist, IsTrue)
err = txn.Commit()
c.Assert(err, IsNil)
_, err = index.FetchValues(make([]types.Datum, 0))
c.Assert(err, NotNil)
}
func (s *testIndexSuite) TestIndex(c *C) {
defer testleak.AfterTest(c)()
index := tables.NewIndex([]byte("i"), "test", 0, false)
// Test ununiq index.
txn, err := s.s.Begin()
c.Assert(err, IsNil)
values := types.MakeDatums(1, 2)
err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
it, err := index.SeekFirst(txn)
c.Assert(err, IsNil)
getValues, h, err := it.Next()
c.Assert(err, IsNil)
c.Assert(getValues, HasLen, 2)
c.Assert(getValues[0].GetInt64(), Equals, int64(1))
c.Assert(getValues[1].GetInt64(), Equals, int64(2))
c.Assert(h, Equals, int64(1))
it.Close()
exist, _, err := index.Exist(txn, values, 100)
c.Assert(err, IsNil)
c.Assert(exist, IsFalse)
exist, _, err = index.Exist(txn, values, 1)
c.Assert(err, IsNil)
c.Assert(exist, IsTrue)
err = index.Delete(txn, values, 1)
c.Assert(err, IsNil)
it, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
err = index.Create(txn, values, 0)
c.Assert(err, IsNil)
_, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, hit, err := index.Seek(txn, values)
c.Assert(err, IsNil)
c.Assert(hit, IsTrue)
err = index.Drop(txn)
c.Assert(err, IsNil)
it, hit, err = index.Seek(txn, values)
c.Assert(err, IsNil)
c.Assert(hit, IsFalse)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
it, err = index.SeekFirst(txn)
c.Assert(err, IsNil)
_, _, err = it.Next()
c.Assert(terror.ErrorEqual(err, io.EOF), IsTrue)
it.Close()
err = txn.Commit()
c.Assert(err, IsNil)
index = tables.NewIndex([]byte("j"), "test", 1, true)
// Test uniq index.
txn, err = s.s.Begin()
c.Assert(err, IsNil)
err = index.Create(txn, values, 1)
c.Assert(err, IsNil)
err = index.Create(txn, values, 2)
c.Assert(err, NotNil)
exist, h, err = index.Exist(txn, values, 1)
c.Assert(err, IsNil)
c.Assert(h, Equals, int64(1))
c.Assert(exist, IsTrue)
exist, h, err = index.Exist(txn, values, 2)
c.Assert(err, NotNil)
c.Assert(h, Equals, int64(1))
c.Assert(exist, IsTrue)
err = txn.Commit()
c.Assert(err, IsNil)
}
