func initTables() {
schem = make(map[string]*schema.Table)
a := schema.NewTable("a")
a.AddColumn("eid", "int", SQLZERO, "")
a.AddColumn("id", "int", SQLZERO, "")
a.AddColumn("name", "varchar(10)", SQLZERO, "")
a.AddColumn("foo", "varchar(10)", SQLZERO, "")
acolumns := []string{"eid", "id", "name", "foo"}
a.Indexes = append(a.Indexes, &schema.Index{"PRIMARY", []string{"eid", "id"}, []uint64{1, 1}, acolumns})
a.Indexes = append(a.Indexes, &schema.Index{"a_name", []string{"eid", "name"}, []uint64{1, 1}, a.Indexes[0].Columns})
a.Indexes = append(a.Indexes, &schema.Index{"b_name", []string{"name"}, []uint64{3}, a.Indexes[0].Columns})
a.Indexes = append(a.Indexes, &schema.Index{"c_name", []string{"name"}, []uint64{2}, a.Indexes[0].Columns})
a.PKColumns = append(a.PKColumns, 0, 1)
a.CacheType = schema.CACHE_RW
schem["a"] = a
b := schema.NewTable("b")
b.AddColumn("eid", "int", SQLZERO, "")
b.AddColumn("id", "int", SQLZERO, "")
bcolumns := []string{"eid", "id"}
b.Indexes = append(a.Indexes, &schema.Index{"PRIMARY", []string{"eid", "id"}, []uint64{1, 1}, bcolumns})
b.PKColumns = append(a.PKColumns, 0, 1)
b.CacheType = schema.CACHE_NONE
schem["b"] = b
c := schema.NewTable("c")
c.AddColumn("eid", "int", SQLZERO, "")
c.AddColumn("id", "int", SQLZERO, "")
c.CacheType = schema.CACHE_NONE
schem["c"] = c
d := schema.NewTable("d")
d.AddColumn("name", "varbinary(10)", SQLZERO, "")
d.AddColumn("id", "int", SQLZERO, "")
d.AddColumn("foo", "varchar(10)", SQLZERO, "")
d.AddColumn("bar", "varchar(10)", SQLZERO, "")
dcolumns := []string{"name"}
d.Indexes = append(d.Indexes, &schema.Index{"PRIMARY", []string{"name"}, []uint64{1}, dcolumns})
d.Indexes = append(d.Indexes, &schema.Index{"d_id", []string{"id"}, []uint64{1}, d.Indexes[0].Columns})
d.Indexes = append(d.Indexes, &schema.Index{"d_bar_never", []string{"bar", "foo"}, []uint64{2, 1}, d.Indexes[0].Columns})
d.Indexes = append(d.Indexes, &schema.Index{"d_bar", []string{"bar", "foo"}, []uint64{3, 1}, d.Indexes[0].Columns})
d.PKColumns = append(d.PKColumns, 0)
d.CacheType = schema.CACHE_RW
schem["d"] = d
e := schema.NewTable("e")
e.AddColumn("eid", "int", SQLZERO, "")
e.AddColumn("id", "int", SQLZERO, "")
ecolumns := []string{"eid", "id"}
e.Indexes = append(e.Indexes, &schema.Index{"PRIMARY", []string{"eid", "id"}, []uint64{1, 1}, ecolumns})
e.PKColumns = append(a.PKColumns, 0, 1)
e.CacheType = schema.CACHE_W
schem["e"] = e
}
// Open initializes the current SchemaInfo for service by loading the necessary info from the specified database.
func (si *SchemaInfo) Open(appParams, dbaParams *sqldb.ConnParams, schemaOverrides []SchemaOverride, cachePool *CachePool, strictMode bool) {
ctx := context.Background()
si.connPool.Open(appParams, dbaParams)
// Get time first because it needs a connection from the pool.
curTime := si.mysqlTime(ctx)
conn := getOrPanic(ctx, si.connPool)
defer conn.Recycle()
if strictMode && !conn.VerifyStrict() {
panic(NewTabletError(ErrFatal, vtrpc.ErrorCode_INTERNAL_ERROR, "Could not verify strict mode"))
}
si.cachePool = cachePool
tables, err := conn.Exec(ctx, baseShowTables, maxTableCount, false)
if err != nil {
panic(PrefixTabletError(ErrFatal, vtrpc.ErrorCode_INTERNAL_ERROR, err, "Could not get table list: "))
}
si.tables = make(map[string]*TableInfo, len(tables.Rows))
// TODO(sougou): Fix this in the parser.
si.tables["dual"] = &TableInfo{Table: schema.NewTable("dual")}
si.tables["DUAL"] = &TableInfo{Table: schema.NewTable("DUAL")}
for _, row := range tables.Rows {
tableName := row[0].String()
tableInfo, err := NewTableInfo(
conn,
tableName,
row[1].String(), // table_type
row[2], // create_time
row[3].String(), // table_comment
si.cachePool,
)
if err != nil {
panic(PrefixTabletError(ErrFatal, vtrpc.ErrorCode_INTERNAL_ERROR, err, fmt.Sprintf("Could not get load table %s: ", tableName)))
}
tableInfo.SetMysqlStats(row[4], row[5], row[6], row[7])
si.tables[tableName] = tableInfo
}
if schemaOverrides != nil {
si.overrides = schemaOverrides
si.override()
}
si.lastChange = curTime
// Clear is not really needed. Doing it for good measure.
si.queries.Clear()
si.ticks.Start(si.Reload)
}
func (si *SchemaInfo) Open(connFactory dbconnpool.CreateConnectionFunc, schemaOverrides []SchemaOverride, cachePool *CachePool, qrs *QueryRules, strictMode bool) {
si.connPool.Open(connFactory)
// Get time first because it needs a connection from the pool.
curTime := si.mysqlTime()
conn := getOrPanic(si.connPool)
defer conn.Recycle()
if strictMode && !conn.(*dbconnpool.PooledDBConnection).VerifyStrict() {
panic(NewTabletError(FATAL, "Could not verify strict mode"))
}
si.cachePool = cachePool
tables, err := conn.ExecuteFetch(base_show_tables, maxTableCount, false)
if err != nil {
panic(NewTabletError(FATAL, "Could not get table list: %v", err))
}
si.tables = make(map[string]*TableInfo, len(tables.Rows))
// TODO(sougou): Fix this in the parser.
si.tables["dual"] = &TableInfo{Table: schema.NewTable("dual")}
si.tables["DUAL"] = &TableInfo{Table: schema.NewTable("DUAL")}
for _, row := range tables.Rows {
tableName := row[0].String()
tableInfo, err := NewTableInfo(
conn,
tableName,
row[1].String(), // table_type
row[2], // create_time
row[3].String(), // table_comment
si.cachePool,
)
if err != nil {
panic(NewTabletError(FATAL, "Could not get load table %s: %v", tableName, err))
}
si.tables[tableName] = tableInfo
}
if schemaOverrides != nil {
si.overrides = schemaOverrides
si.override()
}
si.lastChange = curTime
// Clear is not really needed. Doing it for good measure.
si.queries.Clear()
si.rules = qrs.Copy()
si.ticks.Start(func() { si.Reload() })
}
func TestSchamazHandler(t *testing.T) {
resp := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/schemaz", nil)
tableA := schema.NewTable("a")
tableB := schema.NewTable("b")
tableC := schema.NewTable("c")
tableA.AddColumn("column1", sqltypes.Int64, sqltypes.MakeTrusted(sqltypes.Int32, []byte("0")), "auto_increment")
tableA.AddIndex("index1").AddColumn("index_column", 1000)
tableA.Type = schema.NoType
tableB.AddColumn("column2", sqltypes.VarChar, sqltypes.MakeString([]byte("NULL")), "")
tableB.AddIndex("index2").AddColumn("index_column2", 200)
tableB.Type = schema.Sequence
tables := []*schema.Table{
tableA, tableB, tableC,
}
schemazHandler(tables, resp, req)
body, _ := ioutil.ReadAll(resp.Body)
tableBPattern := []string{
`<td>b</td>`,
`<td>column2: VARCHAR, , NULL<br></td>`,
`<td>index2: \(index_column2,\), \(200,\)<br></td>`,
`<td>sequence</td>`,
}
matched, err := regexp.Match(strings.Join(tableBPattern, `\s*`), body)
if err != nil {
t.Fatalf("schemaz page does not contain table B with error: %v", err)
}
if !matched {
t.Fatalf("schemaz page does not contain table B")
}
tableAPattern := []string{
`<td>a</td>`,
`<td>column1: INT64, autoinc, <br></td>`,
`<td>index1: \(index_column,\), \(1000,\)<br></td>`,
`<td>none</td>`,
}
matched, err = regexp.Match(strings.Join(tableAPattern, `\s*`), body)
if err != nil {
t.Fatalf("schemaz page does not contain table A with error: %v", err)
}
if !matched {
t.Fatalf("schemaz page does not contain table A")
}
}
func NewTableInfo(conn PoolConnection, tableName string, tableType string, createTime sqltypes.Value, comment string, cachePool *CachePool) (ti *TableInfo) {
if tableName == "dual" {
return &TableInfo{Table: schema.NewTable(tableName)}
}
ti = loadTableInfo(conn, tableName)
ti.initRowCache(conn, tableType, createTime, comment, cachePool)
return ti
}
func createTableInfo(name string, cols map[string]string, pKeys []string) TableInfo {
table := schema.NewTable(name)
for colName, colType := range cols {
table.AddColumn(colName, colType, sqltypes.Value{}, "")
}
tableInfo := TableInfo{Table: table}
tableInfo.SetPK(pKeys)
return tableInfo
}
func loadTableInfo(conn *DBConn, tableName string) (ti *TableInfo, err error) {
ti = &TableInfo{Table: schema.NewTable(tableName)}
if err = ti.fetchColumns(conn); err != nil {
return nil, err
}
if err = ti.fetchIndexes(conn); err != nil {
return nil, err
}
return ti, nil
}
func loadTableInfo(conn PoolConnection, tableName string) (ti *TableInfo) {
ti = &TableInfo{Table: schema.NewTable(tableName)}
if !ti.fetchColumns(conn) {
return nil
}
if !ti.fetchIndexes(conn) {
return nil
}
return ti
}
// Open initializes the current SchemaInfo for service by loading the necessary info from the specified database.
func (si *SchemaInfo) Open(appParams, dbaParams *sqldb.ConnParams, schemaOverrides []SchemaOverride, strictMode bool) {
ctx := context.Background()
si.connPool.Open(appParams, dbaParams)
// Get time first because it needs a connection from the pool.
curTime := si.mysqlTime(ctx)
conn := getOrPanic(ctx, si.connPool)
defer conn.Recycle()
if strictMode && !conn.VerifyStrict() {
panic(NewTabletError(ErrFatal, vtrpcpb.ErrorCode_INTERNAL_ERROR, "Could not verify strict mode"))
}
tableData, err := conn.Exec(ctx, baseShowTables, maxTableCount, false)
if err != nil {
panic(PrefixTabletError(ErrFatal, vtrpcpb.ErrorCode_INTERNAL_ERROR, err, "Could not get table list: "))
}
tables := make(map[string]*TableInfo, len(tableData.Rows)+1)
tables["dual"] = &TableInfo{Table: schema.NewTable("dual")}
for _, row := range tableData.Rows {
tableName := row[0].String()
tableInfo, err := NewTableInfo(
conn,
tableName,
row[1].String(), // table_type
row[3].String(), // table_comment
si.cachePool,
)
if err != nil {
si.recordSchemaError(err, tableName)
// Skip over the table that had an error and move on to the next one
continue
}
tableInfo.SetMysqlStats(row[4], row[5], row[6], row[7])
tables[tableName] = tableInfo
}
// Fail if we can't load the schema for any tables, but we know that some tables exist. This points to a configuration problem.
if len(tableData.Rows) != 0 && len(tables) == 1 { // len(tables) is always at least 1 because of the "dual" table
panic(NewTabletError(ErrFail, vtrpcpb.ErrorCode_INTERNAL_ERROR, "could not get schema for any tables"))
}
func() {
si.mu.Lock()
defer si.mu.Unlock()
si.tables = tables
if schemaOverrides != nil {
si.overrides = schemaOverrides
si.override()
}
si.lastChange = curTime
}()
// Clear is not really needed. Doing it for good measure.
si.queries.Clear()
si.ticks.Start(si.Reload)
}
func createTableInfo(
name string, colNames []string, colTypes []string, pKeys []string) TableInfo {
table := schema.NewTable(name)
for i, colName := range colNames {
colType := colTypes[i]
defaultVal := sqltypes.Value{}
if strings.Contains(colType, "int") {
defaultVal = sqltypes.MakeNumeric([]byte("0"))
} else if strings.HasPrefix(colType, "varbinary") {
defaultVal = sqltypes.MakeString([]byte(""))
}
table.AddColumn(colName, colType, defaultVal, "")
}
tableInfo := TableInfo{Table: table}
tableInfo.SetPK(pKeys)
return tableInfo
}
func createTableInfo(
name string, colNames []string, colTypes []querypb.Type, pKeys []string) TableInfo {
table := schema.NewTable(name)
for i, colName := range colNames {
colType := colTypes[i]
defaultVal := sqltypes.Value{}
if sqltypes.IsIntegral(colType) {
defaultVal = sqltypes.MakeTrusted(sqltypes.Int64, []byte("0"))
} else if colType == sqltypes.VarBinary {
defaultVal = sqltypes.MakeString([]byte(""))
}
table.AddColumn(colName, colType, defaultVal, "")
}
tableInfo := TableInfo{Table: table}
tableInfo.SetPK(pKeys)
return tableInfo
}
func TestSchamazHandler(t *testing.T) {
resp := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/schemaz", nil)
tableA := schema.NewTable("a")
tableB := schema.NewTable("b")
tableC := schema.NewTable("c")
tableA.AddColumn("column1", "int", sqltypes.MakeNumeric([]byte("0")), "auto_increment")
tableA.AddIndex("index1").AddColumn("index_column", 1000)
tableA.CacheType = schema.CACHE_RW
tableB.AddColumn("column2", "string", sqltypes.MakeString([]byte("NULL")), "")
tableB.AddIndex("index2").AddColumn("index_column2", 200)
tableB.CacheType = schema.CACHE_W
tableC.AddColumn("column3", "string", sqltypes.MakeString([]byte("")), "")
tableC.AddIndex("index3").AddColumn("index_column3", 500)
tableC.CacheType = schema.CACHE_NONE
tables := []*schema.Table{
tableA, tableB, tableC,
}
schemazHandler(tables, resp, req)
body, _ := ioutil.ReadAll(resp.Body)
tableCPattern := []string{
`<td>c</td>`,
`<td>column3: other, , <br></td>`,
`<td>index3: \(index_column3,\), \(500,\)<br></td>`,
`<td>none</td>`,
}
matched, err := regexp.Match(strings.Join(tableCPattern, `\s*`), body)
if err != nil {
t.Fatalf("schemaz page does not contain table C with error: %v", err)
}
if !matched {
t.Fatalf("schemaz page does not contain table C")
}
tableBPattern := []string{
`<td>b</td>`,
`<td>column2: other, , NULL<br></td>`,
`<td>index2: \(index_column2,\), \(200,\)<br></td>`,
`<td>write-only</td>`,
}
matched, err = regexp.Match(strings.Join(tableBPattern, `\s*`), body)
if err != nil {
t.Fatalf("schemaz page does not contain table B with error: %v", err)
}
if !matched {
t.Fatalf("schemaz page does not contain table B")
}
tableAPattern := []string{
`<td>a</td>`,
`<td>column1: number, autoinc, <br></td>`,
`<td>index1: \(index_column,\), \(1000,\)<br></td>`,
`<td>read-write</td>`,
}
matched, err = regexp.Match(strings.Join(tableAPattern, `\s*`), body)
if err != nil {
t.Fatalf("schemaz page does not contain table A with error: %v", err)
}
if !matched {
t.Fatalf("schemaz page does not contain table A")
}
}
// Open initializes the current SchemaInfo for service by loading the necessary info from the specified database.
func (si *SchemaInfo) Open(dbaParams *sqldb.ConnParams, strictMode bool) {
si.actionMutex.Lock()
defer si.actionMutex.Unlock()
ctx := context.Background()
si.connPool.Open(dbaParams, dbaParams)
// Get time first because it needs a connection from the pool.
curTime := si.mysqlTime(ctx)
conn := getOrPanic(ctx, si.connPool)
defer conn.Recycle()
if strictMode {
if err := conn.VerifyMode(); err != nil {
panic(NewTabletError(vtrpcpb.ErrorCode_INTERNAL_ERROR, err.Error()))
}
}
tableData, err := conn.Exec(ctx, baseShowTables, maxTableCount, false)
if err != nil {
panic(PrefixTabletError(vtrpcpb.ErrorCode_INTERNAL_ERROR, err, "Could not get table list: "))
}
tables := make(map[string]*TableInfo, len(tableData.Rows)+1)
tables["dual"] = &TableInfo{Table: schema.NewTable("dual")}
wg := sync.WaitGroup{}
mu := sync.Mutex{}
for _, row := range tableData.Rows {
wg.Add(1)
go func(row []sqltypes.Value) {
defer wg.Done()
conn := getOrPanic(ctx, si.connPool)
defer conn.Recycle()
tableName := row[0].String()
tableInfo, err := NewTableInfo(
conn,
tableName,
row[1].String(), // table_type
row[3].String(), // table_comment
)
if err != nil {
si.queryServiceStats.InternalErrors.Add("Schema", 1)
log.Errorf("SchemaInfo.Open: failed to create TableInfo for table %s: %v", tableName, err)
// Skip over the table that had an error and move on to the next one
return
}
tableInfo.SetMysqlStats(row[4], row[5], row[6], row[7], row[8])
mu.Lock()
tables[tableName] = tableInfo
mu.Unlock()
}(row)
}
wg.Wait()
// Fail if we can't load the schema for any tables, but we know that some tables exist. This points to a configuration problem.
if len(tableData.Rows) != 0 && len(tables) == 1 { // len(tables) is always at least 1 because of the "dual" table
panic(NewTabletError(vtrpcpb.ErrorCode_INTERNAL_ERROR, "could not get schema for any tables"))
}
func() {
si.mu.Lock()
defer si.mu.Unlock()
si.tables = tables
si.lastChange = curTime
}()
// Clear is not really needed. Doing it for good measure.
si.queries.Clear()
si.ticks.Start(func() {
if err := si.Reload(ctx); err != nil {
log.Errorf("periodic schema reload failed: %v", err)
}
})
}
