func Init(serv SrvTopoServer, cell string, retryDelay time.Duration, retryCount int, timeout time.Duration) { if RpcVTGate != nil { log.Fatalf("VTGate already initialized") } RpcVTGate = &VTGate{ resolver: NewResolver(serv, "VttabletCall", cell, retryDelay, retryCount, timeout), timings: stats.NewMultiTimings("VtgateApi", []string{"Operation", "Keyspace", "DbType"}), errors: stats.NewMultiCounters("VtgateApiErrorCounts", []string{"Operation", "Keyspace", "DbType"}), infoErrors: stats.NewCounters("VtgateInfoErrorCounts"), logExecuteShard: logutil.NewThrottledLogger("ExecuteShard", 5*time.Second), logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second), logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second), logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second), logExecuteBatchShard: logutil.NewThrottledLogger("ExecuteBatchShard", 5*time.Second), logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second), logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second), logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second), logStreamExecuteShard: logutil.NewThrottledLogger("StreamExecuteShard", 5*time.Second), } QPSByOperation = stats.NewRates("QPSByOperation", stats.CounterForDimension(RpcVTGate.timings, "Operation"), 15, 1*time.Minute) QPSByKeyspace = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(RpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) QPSByDbType = stats.NewRates("QPSByDbType", stats.CounterForDimension(RpcVTGate.timings, "DbType"), 15, 1*time.Minute) ErrorsByOperation = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(RpcVTGate.errors, "Operation"), 15, 1*time.Minute) ErrorsByKeyspace = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(RpcVTGate.errors, "Keyspace"), 15, 1*time.Minute) ErrorsByDbType = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(RpcVTGate.errors, "DbType"), 15, 1*time.Minute) for _, f := range RegisterVTGates { f(RpcVTGate) } }
// NewQueryEngine creates a new QueryEngine. // This is a singleton class. // You must call this only once. func NewQueryEngine(config Config) *QueryEngine { qe := &QueryEngine{} qe.schemaInfo = NewSchemaInfo(config.QueryCacheSize, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9)) mysqlStats = stats.NewTimings("Mysql") // Pools qe.cachePool = NewCachePool("Rowcache", config.RowCache, time.Duration(config.QueryTimeout*1e9), time.Duration(config.IdleTimeout*1e9)) qe.connPool = dbconnpool.NewConnectionPool("ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9)) qe.streamConnPool = dbconnpool.NewConnectionPool("StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9)) qe.txPool = dbconnpool.NewConnectionPool("TransactionPool", config.TransactionCap, time.Duration(config.IdleTimeout*1e9)) // connections in pool has to be > transactionCap // Services qe.activeTxPool = NewActiveTxPool("ActiveTransactionPool", time.Duration(config.TransactionTimeout*1e9)) qe.connKiller = NewConnectionKiller(1, time.Duration(config.IdleTimeout*1e9)) qe.activePool = NewActivePool("ActivePool", time.Duration(config.QueryTimeout*1e9), qe.connKiller) qe.consolidator = NewConsolidator() qe.invalidator = NewRowcacheInvalidator(qe) qe.streamQList = NewQueryList(qe.connKiller) // Vars qe.spotCheckFreq = sync2.AtomicInt64(config.SpotCheckRatio * SPOT_CHECK_MULTIPLIER) if config.StrictMode { qe.strictMode.Set(1) } qe.strictTableAcl = config.StrictTableAcl qe.maxResultSize = sync2.AtomicInt64(config.MaxResultSize) qe.streamBufferSize = sync2.AtomicInt64(config.StreamBufferSize) // loggers qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) // Stats stats.Publish("MaxResultSize", stats.IntFunc(qe.maxResultSize.Get)) stats.Publish("StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get)) queryStats = stats.NewTimings("Queries") QPSRates = stats.NewRates("QPS", queryStats, 15, 60*time.Second) waitStats = stats.NewTimings("Waits") killStats = stats.NewCounters("Kills") infoErrors = stats.NewCounters("InfoErrors") errorStats = stats.NewCounters("Errors") internalErrors = stats.NewCounters("InternalErrors") resultStats = stats.NewHistogram("Results", resultBuckets) stats.Publish("RowcacheSpotCheckRatio", stats.FloatFunc(func() float64 { return float64(qe.spotCheckFreq.Get()) / SPOT_CHECK_MULTIPLIER })) spotCheckCount = stats.NewInt("RowcacheSpotCheckCount") return qe }
func Init(serv SrvTopoServer, cell string, retryDelay time.Duration, retryCount int, timeout time.Duration) { if RpcVTGate != nil { log.Fatalf("VTGate already initialized") } RpcVTGate = &VTGate{ resolver: NewResolver(serv, "VttabletCall", cell, retryDelay, retryCount, timeout), timings: stats.NewMapTimings("VtgateApi", []string{"Operation", "Keyspace", "DbType"}), errors: stats.NewMapCounters("VtgateApiErrorCounts", []string{"Operation", "Keyspace", "DbType"}), logExecuteShard: logutil.NewThrottledLogger("ExecuteShard", 5*time.Second), logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second), logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second), logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second), logExecuteBatchShard: logutil.NewThrottledLogger("ExecuteBatchShard", 5*time.Second), logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second), logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second), logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second), logStreamExecuteShard: logutil.NewThrottledLogger("StreamExecuteShard", 5*time.Second), } for _, f := range RegisterVTGates { f(RpcVTGate) } }
// NewQueryEngine creates a new QueryEngine. // This is a singleton class. // You must call this only once. func NewQueryEngine(config Config) *QueryEngine { qe := &QueryEngine{enableAutoCommit: config.EnableAutoCommit} qe.queryServiceStats = NewQueryServiceStats(config.StatsPrefix, config.EnablePublishStats) qe.schemaInfo = NewSchemaInfo( config.QueryCacheSize, config.StatsPrefix, map[string]string{ debugQueryPlansKey: config.DebugURLPrefix + "/query_plans", debugQueryStatsKey: config.DebugURLPrefix + "/query_stats", debugTableStatsKey: config.DebugURLPrefix + "/table_stats", debugSchemaKey: config.DebugURLPrefix + "/schema", }, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) // Pools qe.cachePool = NewCachePool( config.PoolNamePrefix+"Rowcache", config.RowCache, time.Duration(config.IdleTimeout*1e9), config.DebugURLPrefix+"/memcache/", config.EnablePublishStats, qe.queryServiceStats, ) qe.connPool = NewConnPool( config.PoolNamePrefix+"ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) qe.streamConnPool = NewConnPool( config.PoolNamePrefix+"StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) // Services qe.txPool = NewTxPool( config.PoolNamePrefix+"TransactionPool", config.StatsPrefix, config.TransactionCap, time.Duration(config.TransactionTimeout*1e9), time.Duration(config.TxPoolTimeout*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) qe.consolidator = sync2.NewConsolidator() http.Handle(config.DebugURLPrefix+"/consolidations", qe.consolidator) qe.streamQList = NewQueryList() // Vars qe.queryTimeout.Set(time.Duration(config.QueryTimeout * 1e9)) qe.spotCheckFreq = sync2.NewAtomicInt64(int64(config.SpotCheckRatio * spotCheckMultiplier)) if config.StrictMode { qe.strictMode.Set(1) } qe.strictTableAcl = config.StrictTableAcl qe.enableTableAclDryRun = config.EnableTableAclDryRun qe.exemptACL = config.TableAclExemptACL qe.maxResultSize = sync2.NewAtomicInt64(int64(config.MaxResultSize)) qe.maxDMLRows = sync2.NewAtomicInt64(int64(config.MaxDMLRows)) qe.streamBufferSize = sync2.NewAtomicInt64(int64(config.StreamBufferSize)) // Loggers qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) var tableACLAllowedName string var tableACLDeniedName string var tableACLPseudoDeniedName string // Stats if config.EnablePublishStats { stats.Publish(config.StatsPrefix+"MaxResultSize", stats.IntFunc(qe.maxResultSize.Get)) stats.Publish(config.StatsPrefix+"MaxDMLRows", stats.IntFunc(qe.maxDMLRows.Get)) stats.Publish(config.StatsPrefix+"StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get)) stats.Publish(config.StatsPrefix+"QueryTimeout", stats.DurationFunc(qe.queryTimeout.Get)) stats.Publish(config.StatsPrefix+"RowcacheSpotCheckRatio", stats.FloatFunc(func() float64 { return float64(qe.spotCheckFreq.Get()) / spotCheckMultiplier })) stats.Publish(config.StatsPrefix+"TableACLExemptCount", stats.IntFunc(qe.tableaclExemptCount.Get)) tableACLAllowedName = "TableACLAllowed" tableACLDeniedName = "TableACLDenied" tableACLPseudoDeniedName = "TableACLPseudoDenied" } qe.tableaclAllowed = stats.NewMultiCounters(tableACLAllowedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclDenied = stats.NewMultiCounters(tableACLDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclPseudoDenied = stats.NewMultiCounters(tableACLPseudoDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) return qe }
// Init initializes VTGate server. func Init(hc discovery.HealthCheck, topoServer topo.Server, serv SrvTopoServer, schema *planbuilder.Schema, cell string, retryDelay time.Duration, retryCount int, connTimeoutTotal, connTimeoutPerConn, connLife time.Duration, maxInFlight int, testGateway string) { if rpcVTGate != nil { log.Fatalf("VTGate already initialized") } rpcVTGate = &VTGate{ resolver: NewResolver(hc, topoServer, serv, "VttabletCall", cell, retryDelay, retryCount, connTimeoutTotal, connTimeoutPerConn, connLife, testGateway), timings: stats.NewMultiTimings("VtgateApi", []string{"Operation", "Keyspace", "DbType"}), rowsReturned: stats.NewMultiCounters("VtgateApiRowsReturned", []string{"Operation", "Keyspace", "DbType"}), maxInFlight: int64(maxInFlight), inFlight: sync2.NewAtomicInt64(0), logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), logExecuteShards: logutil.NewThrottledLogger("ExecuteShards", 5*time.Second), logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second), logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second), logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second), logExecuteBatchShards: logutil.NewThrottledLogger("ExecuteBatchShards", 5*time.Second), logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second), logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second), logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second), logStreamExecuteShards: logutil.NewThrottledLogger("StreamExecuteShards", 5*time.Second), } // Resuse resolver's scatterConn. rpcVTGate.router = NewRouter(serv, cell, schema, "VTGateRouter", rpcVTGate.resolver.scatterConn) normalErrors = stats.NewMultiCounters("VtgateApiErrorCounts", []string{"Operation", "Keyspace", "DbType"}) infoErrors = stats.NewCounters("VtgateInfoErrorCounts") internalErrors = stats.NewCounters("VtgateInternalErrorCounts") qpsByOperation = stats.NewRates("QPSByOperation", stats.CounterForDimension(rpcVTGate.timings, "Operation"), 15, 1*time.Minute) qpsByKeyspace = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(rpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) qpsByDbType = stats.NewRates("QPSByDbType", stats.CounterForDimension(rpcVTGate.timings, "DbType"), 15, 1*time.Minute) errorsByOperation = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(normalErrors, "Operation"), 15, 1*time.Minute) errorsByKeyspace = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(normalErrors, "Keyspace"), 15, 1*time.Minute) errorsByDbType = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(normalErrors, "DbType"), 15, 1*time.Minute) for _, f := range RegisterVTGates { f(rpcVTGate) } }
vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc" "github.com/youtube/vitess/go/vt/vterrors" ) const ( maxErrLen = 5000 ) // ErrConnPoolClosed is returned / panicked when the connection pool is closed. var ErrConnPoolClosed = NewTabletError( // connection pool being closed is not the query's fault, it can be retried on a // different VtTablet. vtrpcpb.ErrorCode_INTERNAL_ERROR, "connection pool is closed") var logTxPoolFull = logutil.NewThrottledLogger("TxPoolFull", 1*time.Minute) // TabletError is the error type we use in this library type TabletError struct { Message string SQLError int SQLState string // ErrorCode will be used to transmit the error across RPC boundaries ErrorCode vtrpcpb.ErrorCode } // NewTabletError returns a TabletError of the given type func NewTabletError(errCode vtrpcpb.ErrorCode, format string, args ...interface{}) *TabletError { return &TabletError{ Message: printable(fmt.Sprintf(format, args...)), ErrorCode: errCode,
// NewQueryEngine creates a new QueryEngine. // This is a singleton class. // You must call this only once. func NewQueryEngine(checker MySQLChecker, config Config) *QueryEngine { qe := &QueryEngine{config: config} qe.queryServiceStats = NewQueryServiceStats(config.StatsPrefix, config.EnablePublishStats) qe.schemaInfo = NewSchemaInfo( config.StatsPrefix, checker, config.QueryCacheSize, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9), map[string]string{ debugQueryPlansKey: config.DebugURLPrefix + "/query_plans", debugQueryStatsKey: config.DebugURLPrefix + "/query_stats", debugSchemaKey: config.DebugURLPrefix + "/schema", debugQueryRulesKey: config.DebugURLPrefix + "/query_rules", }, config.EnablePublishStats, qe.queryServiceStats, ) qe.connPool = NewConnPool( config.PoolNamePrefix+"ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) qe.streamConnPool = NewConnPool( config.PoolNamePrefix+"StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) qe.txPool = NewTxPool( config.PoolNamePrefix+"TransactionPool", config.StatsPrefix, config.TransactionCap, time.Duration(config.TransactionTimeout*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) // Set the prepared pool capacity to something lower than // tx pool capacity. Those spare connections are needed to // perform metadata state change operations. Without this, // the system can deadlock if all connections get moved to // the TxPreparedPool. prepCap := config.TransactionCap - 2 if prepCap < 0 { // A capacity of 0 means that Prepare will always fail. prepCap = 0 } qe.preparedPool = NewTxPreparedPool(prepCap) qe.twoPC = NewTwoPC() qe.consolidator = sync2.NewConsolidator() http.Handle(config.DebugURLPrefix+"/consolidations", qe.consolidator) qe.streamQList = NewQueryList() if config.StrictMode { qe.strictMode.Set(1) } if config.EnableAutoCommit { qe.autoCommit.Set(1) } qe.strictTableAcl = config.StrictTableAcl qe.enableTableAclDryRun = config.EnableTableAclDryRun if config.TableAclExemptACL != "" { if f, err := tableacl.GetCurrentAclFactory(); err == nil { if exemptACL, err := f.New([]string{config.TableAclExemptACL}); err == nil { log.Infof("Setting Table ACL exempt rule for %v", config.TableAclExemptACL) qe.exemptACL = exemptACL } else { log.Infof("Cannot build exempt ACL for table ACL: %v", err) } } else { log.Infof("Cannot get current ACL Factory: %v", err) } } qe.maxResultSize = sync2.NewAtomicInt64(int64(config.MaxResultSize)) qe.maxDMLRows = sync2.NewAtomicInt64(int64(config.MaxDMLRows)) qe.streamBufferSize = sync2.NewAtomicInt64(int64(config.StreamBufferSize)) qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) var tableACLAllowedName string var tableACLDeniedName string var tableACLPseudoDeniedName string if config.EnablePublishStats { stats.Publish(config.StatsPrefix+"MaxResultSize", stats.IntFunc(qe.maxResultSize.Get)) stats.Publish(config.StatsPrefix+"MaxDMLRows", stats.IntFunc(qe.maxDMLRows.Get)) stats.Publish(config.StatsPrefix+"StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get)) stats.Publish(config.StatsPrefix+"TableACLExemptCount", stats.IntFunc(qe.tableaclExemptCount.Get)) tableACLAllowedName = "TableACLAllowed" tableACLDeniedName = "TableACLDenied" tableACLPseudoDeniedName = "TableACLPseudoDenied" } qe.tableaclAllowed = stats.NewMultiCounters(tableACLAllowedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclDenied = stats.NewMultiCounters(tableACLDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclPseudoDenied = stats.NewMultiCounters(tableACLPseudoDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) return qe }
// NewQueryEngine creates a new QueryEngine. // This is a singleton class. // You must call this only once. func NewQueryEngine(config Config) *QueryEngine { qe := &QueryEngine{enableAutoCommit: config.EnableAutoCommit} qe.queryServiceStats = NewQueryServiceStats(config.StatsPrefix, config.EnablePublishStats) qe.schemaInfo = NewSchemaInfo( config.QueryCacheSize, config.StatsPrefix, map[string]string{ debugQueryPlansKey: config.DebugURLPrefix + "/query_plans", debugQueryStatsKey: config.DebugURLPrefix + "/query_stats", debugTableStatsKey: config.DebugURLPrefix + "/table_stats", debugSchemaKey: config.DebugURLPrefix + "/schema", }, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) // Pools qe.cachePool = NewCachePool( config.PoolNamePrefix+"Rowcache", config.RowCache, time.Duration(config.IdleTimeout*1e9), config.DebugURLPrefix+"/memcache/", config.EnablePublishStats, qe.queryServiceStats, ) qe.connPool = NewConnPool( config.PoolNamePrefix+"ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) qe.streamConnPool = NewConnPool( config.PoolNamePrefix+"StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) // Services qe.txPool = NewTxPool( config.PoolNamePrefix+"TransactionPool", config.StatsPrefix, config.TransactionCap, time.Duration(config.TransactionTimeout*1e9), time.Duration(config.TxPoolTimeout*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, ) qe.consolidator = sync2.NewConsolidator() http.Handle(config.DebugURLPrefix+"/consolidations", qe.consolidator) qe.invalidator = NewRowcacheInvalidator(config.StatsPrefix, qe, config.EnablePublishStats) qe.streamQList = NewQueryList() // Vars qe.queryTimeout.Set(time.Duration(config.QueryTimeout * 1e9)) qe.spotCheckFreq = sync2.AtomicInt64(config.SpotCheckRatio * spotCheckMultiplier) if config.StrictMode { qe.strictMode.Set(1) } qe.strictTableAcl = config.StrictTableAcl qe.maxResultSize = sync2.AtomicInt64(config.MaxResultSize) qe.maxDMLRows = sync2.AtomicInt64(config.MaxDMLRows) qe.streamBufferSize = sync2.AtomicInt64(config.StreamBufferSize) // Loggers qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) // Stats if config.EnablePublishStats { stats.Publish(config.StatsPrefix+"MaxResultSize", stats.IntFunc(qe.maxResultSize.Get)) stats.Publish(config.StatsPrefix+"MaxDMLRows", stats.IntFunc(qe.maxDMLRows.Get)) stats.Publish(config.StatsPrefix+"StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get)) stats.Publish(config.StatsPrefix+"QueryTimeout", stats.DurationFunc(qe.queryTimeout.Get)) stats.Publish(config.StatsPrefix+"RowcacheSpotCheckRatio", stats.FloatFunc(func() float64 { return float64(qe.spotCheckFreq.Get()) / spotCheckMultiplier })) } return qe }
// NewQueryEngine creates a new QueryEngine. // This is a singleton class. // You must call this only once. func NewQueryEngine(checker MySQLChecker, config Config) *QueryEngine { qe := &QueryEngine{config: config} qe.queryServiceStats = NewQueryServiceStats(config.StatsPrefix, config.EnablePublishStats) qe.schemaInfo = NewSchemaInfo( config.StatsPrefix, checker, config.QueryCacheSize, time.Duration(config.SchemaReloadTime*1e9), time.Duration(config.IdleTimeout*1e9), map[string]string{ debugQueryPlansKey: config.DebugURLPrefix + "/query_plans", debugQueryStatsKey: config.DebugURLPrefix + "/query_stats", debugSchemaKey: config.DebugURLPrefix + "/schema", debugQueryRulesKey: config.DebugURLPrefix + "/query_rules", }, config.EnablePublishStats, qe.queryServiceStats, ) qe.connPool = NewConnPool( config.PoolNamePrefix+"ConnPool", config.PoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) qe.streamConnPool = NewConnPool( config.PoolNamePrefix+"StreamConnPool", config.StreamPoolSize, time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) qe.txPool = NewTxPool( config.PoolNamePrefix+"TransactionPool", config.StatsPrefix, config.TransactionCap, time.Duration(config.TransactionTimeout*1e9), time.Duration(config.IdleTimeout*1e9), config.EnablePublishStats, qe.queryServiceStats, checker, ) qe.consolidator = sync2.NewConsolidator() http.Handle(config.DebugURLPrefix+"/consolidations", qe.consolidator) qe.streamQList = NewQueryList() if config.StrictMode { qe.strictMode.Set(1) } if config.EnableAutoCommit { qe.autoCommit.Set(1) } qe.strictTableAcl = config.StrictTableAcl qe.enableTableAclDryRun = config.EnableTableAclDryRun if config.TableAclExemptACL != "" { if f, err := tableacl.GetCurrentAclFactory(); err == nil { if exemptACL, err := f.New([]string{config.TableAclExemptACL}); err == nil { log.Infof("Setting Table ACL exempt rule for %v", config.TableAclExemptACL) qe.exemptACL = exemptACL } else { log.Infof("Cannot build exempt ACL for table ACL: %v", err) } } else { log.Infof("Cannot get current ACL Factory: %v", err) } } qe.maxResultSize = sync2.NewAtomicInt64(int64(config.MaxResultSize)) qe.maxDMLRows = sync2.NewAtomicInt64(int64(config.MaxDMLRows)) qe.streamBufferSize = sync2.NewAtomicInt64(int64(config.StreamBufferSize)) qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second) var tableACLAllowedName string var tableACLDeniedName string var tableACLPseudoDeniedName string if config.EnablePublishStats { stats.Publish(config.StatsPrefix+"MaxResultSize", stats.IntFunc(qe.maxResultSize.Get)) stats.Publish(config.StatsPrefix+"MaxDMLRows", stats.IntFunc(qe.maxDMLRows.Get)) stats.Publish(config.StatsPrefix+"StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get)) stats.Publish(config.StatsPrefix+"TableACLExemptCount", stats.IntFunc(qe.tableaclExemptCount.Get)) tableACLAllowedName = "TableACLAllowed" tableACLDeniedName = "TableACLDenied" tableACLPseudoDeniedName = "TableACLPseudoDenied" } qe.tableaclAllowed = stats.NewMultiCounters(tableACLAllowedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclDenied = stats.NewMultiCounters(tableACLDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) qe.tableaclPseudoDenied = stats.NewMultiCounters(tableACLPseudoDeniedName, []string{"TableName", "TableGroup", "PlanID", "Username"}) return qe }
"log" "strings" "time" "unicode" "github.com/youtube/vitess/go/stats" "github.com/youtube/vitess/go/vt/logutil" ) const ( filteredReplicationUnfriendlyAnnotation = "/* vtgate:: filtered_replication_unfriendly */" ) var ( filteredReplicationUnfriendlyStatementsCount = stats.NewInt("FilteredReplicationUnfriendlyStatementsCount") filteredReplicationUnfriendlyStatementLogger = logutil.NewThrottledLogger("FilteredReplicationUnfriendlyStatement", 5*time.Second) ) // AnnotateIfDML annotates 'sql' based on 'keyspaceIDs' // // If 'sql' is not a DML statement no annotation is added. // If 'sql' is a DML statement and contains exactly one keyspaceID // it is used to annotate 'sql' // Otherwise 'sql' is annotated as replication-unfriendly. func AnnotateIfDML(sql string, keyspaceIDs [][]byte) string { if !IsDML(sql) { return sql } if len(keyspaceIDs) == 1 { return AddKeyspaceID(sql, keyspaceIDs[0], "") }
// Init initializes VTGate server. func Init(ctx context.Context, hc discovery.HealthCheck, topoServer topo.Server, serv topo.SrvTopoServer, cell string, retryCount int, tabletTypesToWait []topodatapb.TabletType) *VTGate { if rpcVTGate != nil { log.Fatalf("VTGate already initialized") } rpcVTGate = &VTGate{ resolver: NewResolver(hc, topoServer, serv, "VttabletCall", cell, retryCount, tabletTypesToWait), timings: stats.NewMultiTimings("VtgateApi", []string{"Operation", "Keyspace", "DbType"}), rowsReturned: stats.NewMultiCounters("VtgateApiRowsReturned", []string{"Operation", "Keyspace", "DbType"}), logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second), logExecuteShards: logutil.NewThrottledLogger("ExecuteShards", 5*time.Second), logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second), logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second), logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second), logExecuteBatchShards: logutil.NewThrottledLogger("ExecuteBatchShards", 5*time.Second), logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second), logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second), logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second), logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second), logStreamExecuteShards: logutil.NewThrottledLogger("StreamExecuteShards", 5*time.Second), logUpdateStream: logutil.NewThrottledLogger("UpdateStream", 5*time.Second), } // vschemaCounters needs to be initialized before planner to // catch the initial load stats. vschemaCounters = stats.NewCounters("VtgateVSchemaCounts") // Resuse resolver's scatterConn. rpcVTGate.router = NewRouter(ctx, serv, cell, "VTGateRouter", rpcVTGate.resolver.scatterConn) normalErrors = stats.NewMultiCounters("VtgateApiErrorCounts", []string{"Operation", "Keyspace", "DbType"}) infoErrors = stats.NewCounters("VtgateInfoErrorCounts") internalErrors = stats.NewCounters("VtgateInternalErrorCounts") qpsByOperation = stats.NewRates("QPSByOperation", stats.CounterForDimension(rpcVTGate.timings, "Operation"), 15, 1*time.Minute) qpsByKeyspace = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(rpcVTGate.timings, "Keyspace"), 15, 1*time.Minute) qpsByDbType = stats.NewRates("QPSByDbType", stats.CounterForDimension(rpcVTGate.timings, "DbType"), 15, 1*time.Minute) errorsByOperation = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(normalErrors, "Operation"), 15, 1*time.Minute) errorsByKeyspace = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(normalErrors, "Keyspace"), 15, 1*time.Minute) errorsByDbType = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(normalErrors, "DbType"), 15, 1*time.Minute) servenv.OnRun(func() { for _, f := range RegisterVTGates { f(rpcVTGate) } }) vtgateOnce.Do(rpcVTGate.registerDebugHealthHandler) return rpcVTGate }