Example #1
0
// backfillIndexInTxn deals with a part of backfilling index data in a Transaction.
// This part of the index data rows is defaultSmallBatchCnt.
func (d *ddl) backfillIndexInTxn(t table.Table, kvIdx table.Index, handles []int64, txn kv.Transaction) (int64, error) {
	idxRecords, err := d.fetchRowColVals(txn, t, handles, kvIdx.Meta())
	if err != nil {
		return 0, errors.Trace(err)
	}

	for _, idxRecord := range idxRecords {
		log.Debug("[ddl] backfill index...", idxRecord.handle)
		err = txn.LockKeys(idxRecord.key)
		if err != nil {
			return 0, errors.Trace(err)
		}

		// Create the index.
		handle, err := kvIdx.Create(txn, idxRecord.vals, idxRecord.handle)
		if err != nil {
			if terror.ErrorEqual(err, kv.ErrKeyExists) && idxRecord.handle == handle {
				// Index already exists, skip it.
				continue
			}
			return 0, errors.Trace(err)
		}
	}
	return idxRecords[len(idxRecords)-1].handle, nil
}
Example #2
0
func checkRecordAndIndex(txn kv.Transaction, t table.Table, idx table.Index) error {
	cols := make([]*table.Column, len(idx.Meta().Columns))
	for i, col := range idx.Meta().Columns {
		cols[i] = t.Cols()[col.Offset]
	}

	startKey := t.RecordKey(0, nil)
	filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) {
		isExist, h2, err := idx.Exist(txn, vals1, h1)
		if terror.ErrorEqual(err, kv.ErrKeyExists) {
			record1 := &RecordData{Handle: h1, Values: vals1}
			record2 := &RecordData{Handle: h2, Values: vals1}
			return false, errDateNotEqual.Gen("index:%v != record:%v", record2, record1)
		}
		if err != nil {
			return false, errors.Trace(err)
		}
		if !isExist {
			record := &RecordData{Handle: h1, Values: vals1}
			return false, errDateNotEqual.Gen("index:%v != record:%v", nil, record)
		}

		return true, nil
	}
	err := iterRecords(txn, t, startKey, cols, filterFunc)

	if err != nil {
		return errors.Trace(err)
	}

	return nil
}
Example #3
0
// BuildIndexForRow implements table.Table BuildIndexForRow interface.
func (t *Table) buildIndexForRow(rm kv.RetrieverMutator, h int64, vals []types.Datum, idx table.Index) error {
	if idx.Meta().State == model.StateDeleteOnly || idx.Meta().State == model.StateDeleteReorganization {
		// If the index is in delete only or write reorganization state, we can not add index.
		return nil
	}

	if _, err := idx.Create(rm, vals, h); err != nil {
		return errors.Trace(err)
	}
	return nil
}
Example #4
0
func (s *testIndexSuite) checkIndexKVExist(c *C, ctx context.Context, t table.Table, handle int64, indexCol table.Index, columnValues []types.Datum, isExist bool) {
	c.Assert(len(indexCol.Meta().Columns), Equals, len(columnValues))

	txn, err := ctx.GetTxn(true)
	c.Assert(err, IsNil)

	exist, _, err := indexCol.Exist(txn, columnValues, handle)
	c.Assert(err, IsNil)
	c.Assert(exist, Equals, isExist)

	err = ctx.CommitTxn()
	c.Assert(err, IsNil)
}
Example #5
0
func checkIndexAndRecord(txn kv.Transaction, t table.Table, idx table.Index) error {
	it, err := idx.SeekFirst(txn)
	if err != nil {
		return errors.Trace(err)
	}
	defer it.Close()

	cols := make([]*table.Column, len(idx.Meta().Columns))
	for i, col := range idx.Meta().Columns {
		cols[i] = t.Cols()[col.Offset]
	}

	for {
		vals1, h, err := it.Next()
		if terror.ErrorEqual(err, io.EOF) {
			break
		} else if err != nil {
			return errors.Trace(err)
		}

		vals2, err := rowWithCols(txn, t, h, cols)
		if terror.ErrorEqual(err, kv.ErrNotExist) {
			record := &RecordData{Handle: h, Values: vals1}
			err = errDateNotEqual.Gen("index:%v != record:%v", record, nil)
		}
		if err != nil {
			return errors.Trace(err)
		}
		if !reflect.DeepEqual(vals1, vals2) {
			record1 := &RecordData{Handle: h, Values: vals1}
			record2 := &RecordData{Handle: h, Values: vals2}
			return errDateNotEqual.Gen("index:%v != record:%v", record1, record2)
		}
	}

	return nil
}
Example #6
0
func (s *testIndexSuite) checkNoneIndex(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, index table.Index, row []types.Datum) {
	t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID)

	columnValues := make([]types.Datum, len(index.Meta().Columns))
	for i, column := range index.Meta().Columns {
		columnValues[i] = row[column.Offset]
	}

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false)
	s.testGetIndex(c, t, index.Meta().Columns[0].Name.L, false)
}
Example #7
0
func (s *testDBSuite) testAddIndex(c *C) {
	done := make(chan struct{}, 1)

	num := 100
	// first add some rows
	for i := 0; i < num; i++ {
		s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
	}

	go func() {
		s.mustExec(c, "create index c3_index on t1 (c3)")
		done <- struct{}{}
	}()

	deletedKeys := make(map[int]struct{})

	ticker := time.NewTicker(s.lease / 2)
	defer ticker.Stop()
LOOP:
	for {
		select {
		case <-done:
			break LOOP
		case <-ticker.C:
			step := 10
			// delete some rows, and add some data
			for i := num; i < num+step; i++ {
				n := rand.Intn(num)
				deletedKeys[n] = struct{}{}
				s.mustExec(c, "delete from t1 where c1 = ?", n)
				s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
			}
			num += step
		}
	}

	// get exists keys
	keys := make([]int, 0, num)
	for i := 0; i < num; i++ {
		if _, ok := deletedKeys[i]; ok {
			continue
		}
		keys = append(keys, i)
	}

	// test index key
	for _, key := range keys {
		rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key)
		matchRows(c, rows, [][]interface{}{{key}})
	}

	// test delete key not in index
	for key := range deletedKeys {
		rows := s.mustQuery(c, "select c1 from t1 where c3 = ?", key)
		matchRows(c, rows, nil)
	}

	// test index range
	for i := 0; i < 100; i++ {
		index := rand.Intn(len(keys) - 3)
		rows := s.mustQuery(c, "select c1 from t1 where c3 >= ? limit 3", keys[index])
		matchRows(c, rows, [][]interface{}{{keys[index]}, {keys[index+1]}, {keys[index+2]}})
	}

	// TODO: support explain in future.
	//rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 100")

	//ay := dumpRows(c, rows)
	//c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsTrue)

	// get all row handles
	ctx := s.s.(context.Context)
	t := s.testGetTable(c, "t1")
	handles := make(map[int64]struct{})
	err := t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) {
		handles[h] = struct{}{}
		return true, nil
	})
	c.Assert(err, IsNil)

	// check in index
	var nidx table.Index
	for _, tidx := range t.Indices() {
		if tidx.Meta().Name.L == "c3_index" {
			nidx = tidx
			break
		}
	}
	// Make sure there is index with name c3_index
	c.Assert(nidx, NotNil)
	c.Assert(nidx.Meta().ID, Greater, int64(0))
	txn, err := ctx.GetTxn(true)
	c.Assert(err, IsNil)
	defer ctx.RollbackTxn()

	it, err := nidx.SeekFirst(txn)
	c.Assert(err, IsNil)
	defer it.Close()

	for {
		_, h, err := it.Next()
		if terror.ErrorEqual(err, io.EOF) {
			break
		}

		c.Assert(err, IsNil)
		_, ok := handles[h]
		c.Assert(ok, IsTrue)
		delete(handles, h)
	}

	c.Assert(handles, HasLen, 0)
}
Example #8
0
func (s *testDBSuite) testDropIndex(c *C) {
	done := make(chan struct{}, 1)

	s.mustExec(c, "delete from t1")

	num := 100
	//  add some rows
	for i := 0; i < num; i++ {
		s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
	}
	t := s.testGetTable(c, "t1")
	var c3idx table.Index
	for _, tidx := range t.Indices() {
		if tidx.Meta().Name.L == "c3_index" {
			c3idx = tidx
			break
		}
	}
	c.Assert(c3idx, NotNil)

	go func() {
		s.mustExec(c, "drop index c3_index on t1")
		done <- struct{}{}
	}()

	ticker := time.NewTicker(s.lease / 2)
	defer ticker.Stop()
LOOP:
	for {
		select {
		case <-done:
			break LOOP
		case <-ticker.C:
			step := 10
			// delete some rows, and add some data
			for i := num; i < num+step; i++ {
				n := rand.Intn(num)
				s.mustExec(c, "update t1 set c2 = 1 where c1 = ?", n)
				s.mustExec(c, "insert into t1 values (?, ?, ?)", i, i, i)
			}
			num += step
		}
	}

	rows := s.mustQuery(c, "explain select c1 from t1 where c3 >= 0")

	ay := dumpRows(c, rows)
	c.Assert(strings.Contains(fmt.Sprintf("%v", ay), "c3_index"), IsFalse)

	// check in index, must no index in kv
	ctx := s.s.(context.Context)

	handles := make(map[int64]struct{})

	t = s.testGetTable(c, "t1")
	var nidx table.Index
	for _, tidx := range t.Indices() {
		if tidx.Meta().Name.L == "c3_index" {
			nidx = tidx
			break
		}
	}
	// Make sure there is no index with name c3_index
	c.Assert(nidx, IsNil)
	idx := tables.NewIndex(t.Meta(), c3idx.Meta())
	txn, err := ctx.GetTxn(true)
	c.Assert(err, IsNil)
	defer ctx.RollbackTxn()

	it, err := idx.SeekFirst(txn)
	c.Assert(err, IsNil)
	defer it.Close()

	for {
		_, h, err := it.Next()
		if terror.ErrorEqual(err, io.EOF) {
			break
		}

		c.Assert(err, IsNil)
		handles[h] = struct{}{}
	}

	c.Assert(handles, HasLen, 0)
}
Example #9
0
func (s *testSuite) testIndex(c *C, tb table.Table, idx table.Index) {
	txn, err := s.store.Begin()
	c.Assert(err, IsNil)

	err = CompareIndexData(txn, tb, idx)
	c.Assert(err, IsNil)

	cnt, err := GetIndexRecordsCount(txn, idx, nil)
	c.Assert(err, IsNil)
	c.Assert(cnt, Equals, int64(2))

	// current index data:
	// index     data (handle, data): (1, 10), (2, 20), (3, 30)
	// index col data (handle, data): (1, 10), (2, 20), (4, 40)
	err = idx.Create(txn, types.MakeDatums(int64(30)), 3)
	c.Assert(err, IsNil)
	col := tb.Cols()[idx.Meta().Columns[0].Offset]
	key := tb.RecordKey(4, col)
	err = tables.SetColValue(txn, key, types.NewDatum(int64(40)))
	c.Assert(err, IsNil)
	err = txn.Commit()
	c.Assert(err, IsNil)

	txn, err = s.store.Begin()
	c.Assert(err, IsNil)
	err = CompareIndexData(txn, tb, idx)
	c.Assert(err, NotNil)
	record1 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(30))}
	diffMsg := newDiffRetError("index", record1, &RecordData{Handle: int64(3), Values: types.MakeDatums(nil)})
	c.Assert(err.Error(), DeepEquals, diffMsg)

	// current index data:
	// index     data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
	// index col data (handle, data): (1, 10), (2, 20), (4, 40), (3, 31)
	err = idx.Create(txn, types.MakeDatums(int64(40)), 4)
	c.Assert(err, IsNil)
	key = tb.RecordKey(3, col)
	err = tables.SetColValue(txn, key, types.NewDatum(int64(31)))
	c.Assert(err, IsNil)
	err = txn.Commit()
	c.Assert(err, IsNil)

	txn, err = s.store.Begin()
	c.Assert(err, IsNil)
	err = CompareIndexData(txn, tb, idx)
	c.Assert(err, NotNil)
	record2 := &RecordData{Handle: int64(3), Values: types.MakeDatums(int64(31))}
	diffMsg = newDiffRetError("index", record1, record2)
	c.Assert(err.Error(), DeepEquals, diffMsg)

	// current index data:
	// index     data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
	// index col data (handle, data): (1, 10), (2, 20), (4, 40), (5, 30)
	key = tb.RecordKey(3, col)
	txn.Delete(key)
	key = tb.RecordKey(5, col)
	err = tables.SetColValue(txn, key, types.NewDatum(int64(30)))
	c.Assert(err, IsNil)
	err = txn.Commit()
	c.Assert(err, IsNil)

	txn, err = s.store.Begin()
	c.Assert(err, IsNil)
	err = checkRecordAndIndex(txn, tb, idx)
	c.Assert(err, NotNil)
	record2 = &RecordData{Handle: int64(5), Values: types.MakeDatums(int64(30))}
	diffMsg = newDiffRetError("index", record1, record2)
	c.Assert(err.Error(), DeepEquals, diffMsg)

	// current index data:
	// index     data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
	// index col data (handle, data): (1, 10), (2, 20), (3, 30)
	key = tb.RecordKey(4, col)
	txn.Delete(key)
	key = tb.RecordKey(3, col)
	err = tables.SetColValue(txn, key, types.NewDatum(int64(30)))
	c.Assert(err, IsNil)
	err = txn.Commit()
	c.Assert(err, IsNil)

	txn, err = s.store.Begin()
	c.Assert(err, IsNil)
	err = CompareIndexData(txn, tb, idx)
	c.Assert(err, NotNil)
	record1 = &RecordData{Handle: int64(4), Values: types.MakeDatums(int64(40))}
	diffMsg = newDiffRetError("index", record1, &RecordData{Handle: int64(4), Values: types.MakeDatums(nil)})
	c.Assert(err.Error(), DeepEquals, diffMsg)

	// current index data:
	// index     data (handle, data): (1, 10), (2, 20), (3, 30)
	// index col data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40)
	err = idx.Delete(txn, types.MakeDatums(int64(40)), 4)
	c.Assert(err, IsNil)
	key = tb.RecordKey(4, col)
	err = tables.SetColValue(txn, key, types.NewDatum(int64(40)))
	c.Assert(err, IsNil)
	err = txn.Commit()
	c.Assert(err, IsNil)

	txn, err = s.store.Begin()
	c.Assert(err, IsNil)
	err = CompareIndexData(txn, tb, idx)
	c.Assert(err, NotNil)
	diffMsg = newDiffRetError("index", nil, record1)
	c.Assert(err.Error(), DeepEquals, diffMsg)
}
Example #10
0
func (s *testIndexSuite) checkPublicIndex(c *C, ctx context.Context, d *ddl, tblInfo *model.TableInfo, handle int64, index table.Index, row []types.Datum) {
	t := testGetTable(c, d, s.dbInfo.ID, tblInfo.ID)

	_, err := ctx.GetTxn(true)
	c.Assert(err, IsNil)

	i := int64(0)
	err = t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) {
		c.Assert(data, DeepEquals, row)
		i++
		return true, nil
	})
	c.Assert(err, IsNil)
	c.Assert(i, Equals, int64(1))

	columnValues := make([]types.Datum, len(index.Meta().Columns))
	for i, column := range index.Meta().Columns {
		columnValues[i] = row[column.Offset]
	}

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true)

	// Test add a new row.
	_, err = ctx.GetTxn(true)
	c.Assert(err, IsNil)

	newRow := types.MakeDatums(int64(11), int64(22), int64(33))
	handle, err = t.AddRecord(ctx, newRow)
	c.Assert(err, IsNil)

	_, err = ctx.GetTxn(true)
	c.Assert(err, IsNil)

	rows := [][]types.Datum{row, newRow}

	i = int64(0)
	t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) {
		c.Assert(data, DeepEquals, rows[i])
		i++
		return true, nil
	})
	c.Assert(i, Equals, int64(2))

	for i, column := range index.Meta().Columns {
		columnValues[i] = newRow[column.Offset]
	}

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true)

	// Test update a new row.
	_, err = ctx.GetTxn(true)
	c.Assert(err, IsNil)

	newUpdateRow := types.MakeDatums(int64(44), int64(55), int64(66))
	touched := map[int]bool{0: true, 1: true, 2: true}
	err = t.UpdateRecord(ctx, handle, newRow, newUpdateRow, touched)
	c.Assert(err, IsNil)

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false)

	for i, column := range index.Meta().Columns {
		columnValues[i] = newUpdateRow[column.Offset]
	}

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, true)

	// Test remove a row.
	_, err = ctx.GetTxn(true)
	c.Assert(err, IsNil)

	err = t.RemoveRecord(ctx, handle, newUpdateRow)
	c.Assert(err, IsNil)

	_, err = ctx.GetTxn(true)
	c.Assert(err, IsNil)

	i = int64(0)
	t.IterRecords(ctx, t.FirstKey(), t.Cols(), func(h int64, data []types.Datum, cols []*table.Column) (bool, error) {
		i++
		return true, nil
	})
	c.Assert(i, Equals, int64(1))

	s.checkIndexKVExist(c, ctx, t, handle, index, columnValues, false)
	s.testGetIndex(c, t, index.Meta().Columns[0].Name.L, true)
}
Example #11
0
func (b *executorBuilder) buildIndexScan(v *plan.IndexScan) Executor {
	txn, err := b.ctx.GetTxn(false)
	if err != nil {
		b.err = err
		return nil
	}
	tbl, _ := b.is.TableByID(v.Table.ID)
	client := txn.GetClient()
	supportDesc := client.SupportRequestType(kv.ReqTypeIndex, kv.ReqSubTypeDesc)
	var memDB bool
	switch v.Fields()[0].DBName.L {
	case "information_schema", "performance_schema":
		memDB = true
	}
	if !memDB && client.SupportRequestType(kv.ReqTypeIndex, 0) {
		log.Debug("xapi select index")
		e := &XSelectIndexExec{
			table:       tbl,
			ctx:         b.ctx,
			indexPlan:   v,
			supportDesc: supportDesc,
		}
		where, remained := b.conditionsToPBExpr(client, v.FilterConditions, v.TableName)
		if where != nil {
			e.where = where
		}
		var ex Executor
		if txn.IsReadOnly() {
			ex = e
		} else {
			ex = b.buildUnionScanExec(e)
		}
		return b.buildFilter(ex, remained)
	}

	var idx table.Index
	for _, val := range tbl.Indices() {
		if val.Meta().Name.L == v.Index.Name.L {
			idx = val
			break
		}
	}
	e := &IndexScanExec{
		tbl:         tbl,
		tableAsName: v.TableAsName,
		idx:         idx,
		fields:      v.Fields(),
		ctx:         b.ctx,
		Desc:        v.Desc,
		valueTypes:  make([]*types.FieldType, len(idx.Meta().Columns)),
	}

	for i, ic := range idx.Meta().Columns {
		col := tbl.Cols()[ic.Offset]
		e.valueTypes[i] = &col.FieldType
	}

	e.Ranges = make([]*IndexRangeExec, len(v.Ranges))
	for i, val := range v.Ranges {
		e.Ranges[i] = b.buildIndexRange(e, val)
	}
	x := b.buildFilter(e, v.FilterConditions)
	if v.Desc {
		x = &ReverseExec{Src: x}
	}
	return x
}