Beispiel #1
0
func (qs *QuerySplitter) parseInt(pkMinMax *mproto.QueryResult) ([]sqltypes.Value, error) {
	boundaries := []sqltypes.Value{}
	minNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][0].Raw())
	maxNumeric := sqltypes.MakeNumeric(pkMinMax.Rows[0][1].Raw())
	if pkMinMax.Rows[0][0].Raw()[0] == '-' {
		// signed values, use int64
		min, err := minNumeric.ParseInt64()
		if err != nil {
			return nil, err
		}
		max, err := maxNumeric.ParseInt64()
		if err != nil {
			return nil, err
		}
		interval := (max - min) / int64(qs.splitCount)
		if interval == 0 {
			return nil, err
		}
		qs.rowCount = interval
		for i := int64(1); i < int64(qs.splitCount); i++ {
			v, err := sqltypes.BuildValue(min + interval*i)
			if err != nil {
				return nil, err
			}
			boundaries = append(boundaries, v)
		}
		return boundaries, nil
	}
	// unsigned values, use uint64
	min, err := minNumeric.ParseUint64()
	if err != nil {
		return nil, err
	}
	max, err := maxNumeric.ParseUint64()
	if err != nil {
		return nil, err
	}
	interval := (max - min) / uint64(qs.splitCount)
	if interval == 0 {
		return nil, err
	}
	qs.rowCount = int64(interval)
	for i := uint64(1); i < uint64(qs.splitCount); i++ {
		v, err := sqltypes.BuildValue(min + interval*i)
		if err != nil {
			return nil, err
		}
		boundaries = append(boundaries, v)
	}
	return boundaries, nil
}
Beispiel #2
0
// Split will split the rows into subset for each distribution
func (rs *RowSplitter) Split(result [][][]sqltypes.Value, rows [][]sqltypes.Value) error {
	if rs.Type == key.KIT_UINT64 {
		for _, row := range rows {
			v := sqltypes.MakeNumeric(row[rs.ValueIndex].Raw())
			i, err := v.ParseUint64()
			if err != nil {
				return fmt.Errorf("Non numerical value: %v", err)
			}
			k := key.Uint64Key(i).KeyspaceId()
			for i, kr := range rs.KeyRanges {
				if kr.Contains(k) {
					result[i] = append(result[i], row)
					break
				}
			}
		}
	} else {
		for _, row := range rows {
			k := key.KeyspaceId(row[rs.ValueIndex].Raw())
			for i, kr := range rs.KeyRanges {
				if kr.Contains(k) {
					result[i] = append(result[i], row)
					break
				}
			}
		}
	}
	return nil
}
Beispiel #3
0
// makeValueString returns a string that contains all the passed-in rows
// as an insert SQL command's parameters.
func makeValueString(fields []mproto.Field, rows [][]sqltypes.Value) string {
	buf := bytes.Buffer{}
	for i, row := range rows {
		if i > 0 {
			buf.Write([]byte(",("))
		} else {
			buf.WriteByte('(')
		}
		for j, value := range row {
			if j > 0 {
				buf.WriteByte(',')
			}
			// convert value back to its original type
			if !value.IsNull() {
				switch fields[j].Type {
				case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24:
					value = sqltypes.MakeNumeric(value.Raw())
				case mproto.VT_FLOAT, mproto.VT_DOUBLE:
					value = sqltypes.MakeFractional(value.Raw())
				}
			}
			value.EncodeSql(&buf)
		}
		buf.WriteByte(')')
	}
	return buf.String()
}
Beispiel #4
0
func BuildValue(bytes []byte, fieldType uint32) sqltypes.Value {
	if bytes == nil {
		return sqltypes.NULL
	}
	switch fieldType {
	case C.MYSQL_TYPE_DECIMAL, C.MYSQL_TYPE_FLOAT, C.MYSQL_TYPE_DOUBLE, C.MYSQL_TYPE_NEWDECIMAL:
		return sqltypes.MakeFractional(bytes)
	case C.MYSQL_TYPE_TIMESTAMP:
		return sqltypes.MakeString(bytes)
	}
	// The below condition represents the following list of values:
	// C.MYSQL_TYPE_TINY, C.MYSQL_TYPE_SHORT, C.MYSQL_TYPE_LONG, C.MYSQL_TYPE_LONGLONG, C.MYSQL_TYPE_INT24, C.MYSQL_TYPE_YEAR:
	if fieldType <= C.MYSQL_TYPE_INT24 || fieldType == C.MYSQL_TYPE_YEAR {
		return sqltypes.MakeNumeric(bytes)
	}
	return sqltypes.MakeString(bytes)
}
Beispiel #5
0
func (rc *RowCache) decodeRow(b []byte) (row []sqltypes.Value) {
	rowlen := pack.Uint32(b)
	data := b[4+rowlen*4:]
	row = make([]sqltypes.Value, rowlen)
	for i := range row {
		length := pack.Uint32(b[4+i*4:])
		if length == 0xFFFFFFFF {
			continue
		}
		if length > uint32(len(data)) {
			// Corrupt data
			return nil
		}
		if rc.tableInfo.Columns[i].Category == schema.CAT_NUMBER {
			row[i] = sqltypes.MakeNumeric(data[:length])
		} else {
			row[i] = sqltypes.MakeString(data[:length])
		}
		data = data[length:]
	}
	return row
}
Beispiel #6
0
func (vc *vcursor) Execute(query *tproto.BoundQuery) (*mproto.QueryResult, error) {
	vc.query = query
	switch {
	case strings.HasPrefix(query.Sql, "select"):
		return &mproto.QueryResult{
			Fields: []mproto.Field{{
				Type: mproto.VT_LONG,
			}},
			Rows: [][]sqltypes.Value{
				[]sqltypes.Value{
					sqltypes.MakeNumeric([]byte("1")),
				},
			},
			RowsAffected: 1,
		}, nil
	case strings.HasPrefix(query.Sql, "insert"):
		return &mproto.QueryResult{InsertId: 1}, nil
	case strings.HasPrefix(query.Sql, "delete"):
		return &mproto.QueryResult{}, nil
	}
	panic("unexpected")
}
Beispiel #7
0
func (ta *Table) AddColumn(name string, columnType string, defval sqltypes.Value, extra string) {
	index := len(ta.Columns)
	ta.Columns = append(ta.Columns, TableColumn{Name: name})
	if strings.Contains(columnType, "int") {
		ta.Columns[index].Category = CAT_NUMBER
	} else if strings.HasPrefix(columnType, "varbinary") {
		ta.Columns[index].Category = CAT_VARBINARY
	} else {
		ta.Columns[index].Category = CAT_OTHER
	}
	if extra == "auto_increment" {
		ta.Columns[index].IsAuto = true
		// Ignore default value, if any
		return
	}
	if defval.IsNull() {
		return
	}
	if ta.Columns[index].Category == CAT_NUMBER {
		ta.Columns[index].Default = sqltypes.MakeNumeric(defval.Raw())
	} else {
		ta.Columns[index].Default = sqltypes.MakeString(defval.Raw())
	}
}
Beispiel #8
0
func TestParsedQuery(t *testing.T) {
	tcases := []struct {
		desc     string
		query    string
		bindVars map[string]interface{}
		output   string
	}{
		{
			"no subs",
			"select * from a where id = 2",
			map[string]interface{}{
				"id": 1,
			},
			"select * from a where id = 2",
		}, {
			"simple bindvar sub",
			"select * from a where id1 = :id1 and id2 = :id2",
			map[string]interface{}{
				"id1": 1,
				"id2": nil,
			},
			"select * from a where id1 = 1 and id2 = null",
		}, {
			"missing bind var",
			"select * from a where id1 = :id1 and id2 = :id2",
			map[string]interface{}{
				"id1": 1,
			},
			"missing bind var id2",
		}, {
			"unencodable bind var",
			"select * from a where id1 = :id",
			map[string]interface{}{
				"id": make([]int, 1),
			},
			"unsupported bind variable type []int: [0]",
		}, {
			"list inside bind vars",
			"select * from a where id in (:vals)",
			map[string]interface{}{
				"vals": []sqltypes.Value{
					sqltypes.MakeNumeric([]byte("1")),
					sqltypes.MakeString([]byte("aa")),
				},
			},
			"select * from a where id in (1, 'aa')",
		}, {
			"two lists inside bind vars",
			"select * from a where id in (:vals)",
			map[string]interface{}{
				"vals": [][]sqltypes.Value{
					[]sqltypes.Value{
						sqltypes.MakeNumeric([]byte("1")),
						sqltypes.MakeString([]byte("aa")),
					},
					[]sqltypes.Value{
						sqltypes.Value{},
						sqltypes.MakeString([]byte("bb")),
					},
				},
			},
			"select * from a where id in ((1, 'aa'), (null, 'bb'))",
		}, {
			"list bind vars",
			"select * from a where id in ::vals",
			map[string]interface{}{
				"vals": []interface{}{
					1,
					"aa",
				},
			},
			"select * from a where id in (1, 'aa')",
		}, {
			"list bind vars single argument",
			"select * from a where id in ::vals",
			map[string]interface{}{
				"vals": []interface{}{
					1,
				},
			},
			"select * from a where id in (1)",
		}, {
			"list bind vars 0 arguments",
			"select * from a where id in ::vals",
			map[string]interface{}{
				"vals": []interface{}{},
			},
			"empty list supplied for vals",
		}, {
			"non-list bind var supplied",
			"select * from a where id in ::vals",
			map[string]interface{}{
				"vals": 1,
			},
			"unexpected list arg type int for key vals",
		}, {
			"list bind var for non-list",
			"select * from a where id = :vals",
			map[string]interface{}{
				"vals": []interface{}{1},
			},
			"unexpected arg type []interface {} for key vals",
		}, {
			"single column tuple equality",
			// We have to use an incorrect construct to get around the parser.
			"select * from a where b = :equality",
			map[string]interface{}{
				"equality": TupleEqualityList{
					Columns: []string{"pk"},
					Rows: [][]sqltypes.Value{
						[]sqltypes.Value{sqltypes.MakeNumeric([]byte("1"))},
						[]sqltypes.Value{sqltypes.MakeString([]byte("aa"))},
					},
				},
			},
			"select * from a where b = pk in (1, 'aa')",
		}, {
			"multi column tuple equality",
			"select * from a where b = :equality",
			map[string]interface{}{
				"equality": TupleEqualityList{
					Columns: []string{"pk1", "pk2"},
					Rows: [][]sqltypes.Value{
						[]sqltypes.Value{
							sqltypes.MakeNumeric([]byte("1")),
							sqltypes.MakeString([]byte("aa")),
						},
						[]sqltypes.Value{
							sqltypes.MakeNumeric([]byte("2")),
							sqltypes.MakeString([]byte("bb")),
						},
					},
				},
			},
			"select * from a where b = (pk1 = 1 and pk2 = 'aa') or (pk1 = 2 and pk2 = 'bb')",
		}, {
			"0 rows",
			"select * from a where b = :equality",
			map[string]interface{}{
				"equality": TupleEqualityList{
					Columns: []string{"pk"},
					Rows:    [][]sqltypes.Value{},
				},
			},
			"cannot encode with 0 rows",
		}, {
			"values don't match column count",
			"select * from a where b = :equality",
			map[string]interface{}{
				"equality": TupleEqualityList{
					Columns: []string{"pk"},
					Rows: [][]sqltypes.Value{
						[]sqltypes.Value{
							sqltypes.MakeNumeric([]byte("1")),
							sqltypes.MakeString([]byte("aa")),
						},
					},
				},
			},
			"values don't match column count",
		},
	}

	for _, tcase := range tcases {
		tree, err := Parse(tcase.query)
		if err != nil {
			t.Errorf("parse failed for %s: %v", tcase.desc, err)
			continue
		}
		buf := NewTrackedBuffer(nil)
		buf.Myprintf("%v", tree)
		pq := buf.ParsedQuery()
		bytes, err := pq.GenerateQuery(tcase.bindVars)
		var got string
		if err != nil {
			got = err.Error()
		} else {
			got = string(bytes)
		}
		if got != tcase.output {
			t.Errorf("for test case: %s, got: '%s', want '%s'", tcase.desc, got, tcase.output)
		}
	}
}
Beispiel #9
0
		encoded: "E\x00\x00\x00\x04Fields\x00\x05\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
	},
	// Only fields set
	{
		qr: QueryResult{
			Fields: []Field{
				{Name: "foo", Type: 1},
			},
		},
		encoded: "i\x00\x00\x00\x04Fields\x00)\x00\x00\x00\x030\x00!\x00\x00\x00\x05Name\x00\x03\x00\x00\x00\x00foo\x12Type\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x00\x05\x00\x00\x00\x00\x00",
	},
	// Only rows, no fields
	{
		qr: QueryResult{
			Rows: [][]sqltypes.Value{
				{sqltypes.MakeString([]byte("abcd")), sqltypes.MakeNumeric([]byte("1234")), sqltypes.MakeFractional([]byte("1.234"))},
			},
		},
		encoded: "r\x00\x00\x00\x04Fields\x00\x05\x00\x00\x00\x00?RowsAffected\x00\x00\x00\x00\x00\x00\x00\x00\x00?InsertId\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04Rows\x002\x00\x00\x00\x040\x00*\x00\x00\x00\x050\x00\x04\x00\x00\x00\x00abcd\x051\x00\x04\x00\x00\x00\x001234\x052\x00\x05\x00\x00\x00\x001.234\x00\x00\x00",
	},
	// one row and one field
	{
		qr: QueryResult{
			Fields: []Field{
				{Name: "foo", Type: 1},
			},
			Rows: [][]sqltypes.Value{
				{sqltypes.MakeString([]byte("abcd")), sqltypes.MakeNumeric([]byte("1234")), sqltypes.MakeFractional([]byte("1.234")), sqltypes.Value{}},
			},
		},
		encoded: "",
Beispiel #10
0
// findChunks returns an array of chunks to use for splitting up a table
// into multiple data chunks. It only works for tables with a primary key
// (and the primary key first column is an integer type).
// The array will always look like:
// "", "value1", "value2", ""
// A non-split tablet will just return:
// "", ""
func findChunks(wr *wrangler.Wrangler, ti *topo.TabletInfo, td *myproto.TableDefinition, minTableSizeForSplit uint64, sourceReaderCount int) ([]string, error) {
	result := []string{"", ""}

	// eliminate a few cases we don't split tables for
	if len(td.PrimaryKeyColumns) == 0 {
		// no primary key, what can we do?
		return result, nil
	}
	if td.DataLength < minTableSizeForSplit {
		// table is too small to split up
		return result, nil
	}

	// get the min and max of the leading column of the primary key
	query := fmt.Sprintf("SELECT MIN(%v), MAX(%v) FROM %v.%v", td.PrimaryKeyColumns[0], td.PrimaryKeyColumns[0], ti.DbName(), td.Name)
	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
	qr, err := wr.TabletManagerClient().ExecuteFetch(ctx, ti, query, 1, true, false)
	if err != nil {
		wr.Logger().Infof("Not splitting table %v into multiple chunks: %v", td.Name, err)
		return result, nil
	}
	cancel()
	if len(qr.Rows) != 1 {
		wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot get min and max", td.Name)
		return result, nil
	}
	if qr.Rows[0][0].IsNull() || qr.Rows[0][1].IsNull() {
		wr.Logger().Infof("Not splitting table %v into multiple chunks, min or max is NULL: %v %v", td.Name, qr.Rows[0][0], qr.Rows[0][1])
		return result, nil
	}
	switch qr.Fields[0].Type {
	case mproto.VT_TINY, mproto.VT_SHORT, mproto.VT_LONG, mproto.VT_LONGLONG, mproto.VT_INT24:
		minNumeric := sqltypes.MakeNumeric(qr.Rows[0][0].Raw())
		maxNumeric := sqltypes.MakeNumeric(qr.Rows[0][1].Raw())
		if qr.Rows[0][0].Raw()[0] == '-' {
			// signed values, use int64
			min, err := minNumeric.ParseInt64()
			if err != nil {
				wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
				return result, nil
			}
			max, err := maxNumeric.ParseInt64()
			if err != nil {
				wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
				return result, nil
			}
			interval := (max - min) / int64(sourceReaderCount)
			if interval == 0 {
				wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
				return result, nil
			}

			result = make([]string, sourceReaderCount+1)
			result[0] = ""
			result[sourceReaderCount] = ""
			for i := int64(1); i < int64(sourceReaderCount); i++ {
				result[i] = fmt.Sprintf("%v", min+interval*i)
			}
			return result, nil
		}

		// unsigned values, use uint64
		min, err := minNumeric.ParseUint64()
		if err != nil {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, minNumeric, err)
			return result, nil
		}
		max, err := maxNumeric.ParseUint64()
		if err != nil {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, maxNumeric, err)
			return result, nil
		}
		interval := (max - min) / uint64(sourceReaderCount)
		if interval == 0 {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
			return result, nil
		}

		result = make([]string, sourceReaderCount+1)
		result[0] = ""
		result[sourceReaderCount] = ""
		for i := uint64(1); i < uint64(sourceReaderCount); i++ {
			result[i] = fmt.Sprintf("%v", min+interval*i)
		}
		return result, nil

	case mproto.VT_FLOAT, mproto.VT_DOUBLE:
		min, err := strconv.ParseFloat(qr.Rows[0][0].String(), 64)
		if err != nil {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert min: %v %v", td.Name, qr.Rows[0][0], err)
			return result, nil
		}
		max, err := strconv.ParseFloat(qr.Rows[0][1].String(), 64)
		if err != nil {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, cannot convert max: %v %v", td.Name, qr.Rows[0][1].String(), err)
			return result, nil
		}
		interval := (max - min) / float64(sourceReaderCount)
		if interval == 0 {
			wr.Logger().Infof("Not splitting table %v into multiple chunks, interval=0: %v %v", td.Name, max, min)
			return result, nil
		}

		result = make([]string, sourceReaderCount+1)
		result[0] = ""
		result[sourceReaderCount] = ""
		for i := 1; i < sourceReaderCount; i++ {
			result[i] = fmt.Sprintf("%v", min+interval*float64(i))
		}
		return result, nil
	}

	wr.Logger().Infof("Not splitting table %v into multiple chunks, primary key not numeric", td.Name)
	return result, nil
}