func NewTopoReader(ts vtgate.SrvTopoServer) *TopoReader {
return &TopoReader{
ts: ts,
queryCount: stats.NewCounters("TopoReaderRpcQueryCount"),
errorCount: stats.NewCounters("TopoReaderRpcErrorCount"),
}
}
func newZkrStats() *zkrStats {
zs := &zkrStats{}
zs.zkReads = stats.NewCounters("ZkReaderZkReads")
zs.cacheReads = stats.NewCounters("ZkReaderCacheReads")
zs.staleReads = stats.NewCounters("ZkReaderStaleReads")
zs.nodeNotFoundErrors = stats.NewCounters("ZkReaderNodeNotFoundErrors")
zs.otherErrors = stats.NewCounters("ZkReaderOtherErrors")
return zs
}
func Init(serv SrvTopoServer, schema *planbuilder.Schema, cell string, retryDelay time.Duration, retryCount int, timeout time.Duration, maxInFlight int) {
if RpcVTGate != nil {
log.Fatalf("VTGate already initialized")
}
RpcVTGate = &VTGate{
resolver: NewResolver(serv, "VttabletCall", cell, retryDelay, retryCount, timeout),
timings: stats.NewMultiTimings("VtgateApi", []string{"Operation", "Keyspace", "DbType"}),
rowsReturned: stats.NewMultiCounters("VtgateApiRowsReturned", []string{"Operation", "Keyspace", "DbType"}),
maxInFlight: int64(maxInFlight),
inFlight: 0,
logExecuteShard: logutil.NewThrottledLogger("ExecuteShard", 5*time.Second),
logExecuteKeyspaceIds: logutil.NewThrottledLogger("ExecuteKeyspaceIds", 5*time.Second),
logExecuteKeyRanges: logutil.NewThrottledLogger("ExecuteKeyRanges", 5*time.Second),
logExecuteEntityIds: logutil.NewThrottledLogger("ExecuteEntityIds", 5*time.Second),
logExecuteBatchShard: logutil.NewThrottledLogger("ExecuteBatchShard", 5*time.Second),
logExecuteBatchKeyspaceIds: logutil.NewThrottledLogger("ExecuteBatchKeyspaceIds", 5*time.Second),
logStreamExecuteKeyspaceIds: logutil.NewThrottledLogger("StreamExecuteKeyspaceIds", 5*time.Second),
logStreamExecuteKeyRanges: logutil.NewThrottledLogger("StreamExecuteKeyRanges", 5*time.Second),
logStreamExecuteShard: logutil.NewThrottledLogger("StreamExecuteShard", 5*time.Second),
}
// Resuse resolver's scatterConn.
RpcVTGate.router = NewRouter(serv, cell, schema, "VTGateRouter", RpcVTGate.resolver.scatterConn)
normalErrors = stats.NewMultiCounters("VtgateApiErrorCounts", []string{"Operation", "Keyspace", "DbType"})
infoErrors = stats.NewCounters("VtgateInfoErrorCounts")
internalErrors = stats.NewCounters("VtgateInternalErrorCounts")
QPSByOperation = stats.NewRates("QPSByOperation", stats.CounterForDimension(RpcVTGate.timings, "Operation"), 15, 1*time.Minute)
QPSByKeyspace = stats.NewRates("QPSByKeyspace", stats.CounterForDimension(RpcVTGate.timings, "Keyspace"), 15, 1*time.Minute)
QPSByDbType = stats.NewRates("QPSByDbType", stats.CounterForDimension(RpcVTGate.timings, "DbType"), 15, 1*time.Minute)
ErrorsByOperation = stats.NewRates("ErrorsByOperation", stats.CounterForDimension(normalErrors, "Operation"), 15, 1*time.Minute)
ErrorsByKeyspace = stats.NewRates("ErrorsByKeyspace", stats.CounterForDimension(normalErrors, "Keyspace"), 15, 1*time.Minute)
ErrorsByDbType = stats.NewRates("ErrorsByDbType", stats.CounterForDimension(normalErrors, "DbType"), 15, 1*time.Minute)
for _, f := range RegisterVTGates {
f(RpcVTGate)
}
}
// NewResilientSrvTopoServer creates a new ResilientSrvTopoServer
// based on the provided SrvTopoServer.
func NewResilientSrvTopoServer(base topo.Server, counterPrefix string) *ResilientSrvTopoServer {
return &ResilientSrvTopoServer{
topoServer: base,
cacheTTL: *srvTopoCacheTTL,
enableRemoteMaster: *enableRemoteMaster,
counts: stats.NewCounters(counterPrefix + "Counts"),
srvKeyspaceNamesCache: make(map[string]*srvKeyspaceNamesEntry),
srvKeyspaceCache: make(map[string]*srvKeyspaceEntry),
endPointsCache: make(map[string]*endPointsEntry),
endPointCounters: newEndPointCounters(counterPrefix),
}
}
// Package streamlog provides a non-blocking message broadcaster.
package streamlog
import (
"io"
"net/http"
"net/url"
"sync"
log "github.com/golang/glog"
"github.com/henryanand/vitess/go/acl"
"github.com/henryanand/vitess/go/stats"
)
var (
sendCount = stats.NewCounters("StreamlogSend")
deliveredCount = stats.NewMultiCounters("StreamlogDelivered", []string{"Log", "Subscriber"})
deliveryDropCount = stats.NewMultiCounters("StreamlogDeliveryDroppedMessages", []string{"Log", "Subscriber"})
)
type subscriber struct {
name string
}
// StreamLogger is a non-blocking broadcaster of messages.
// Subscribers can use channels or HTTP.
type StreamLogger struct {
name string
dataQueue chan interface{}
mu sync.Mutex
subscribed map[chan interface{}]subscriber
)
/* API and config for UpdateStream Service */
const (
DISABLED int64 = iota
ENABLED
)
var usStateNames = map[int64]string{
ENABLED: "Enabled",
DISABLED: "Disabled",
}
var (
streamCount = stats.NewCounters("UpdateStreamStreamCount")
updateStreamErrors = stats.NewCounters("UpdateStreamErrors")
updateStreamEvents = stats.NewCounters("UpdateStreamEvents")
keyrangeStatements = stats.NewInt("UpdateStreamKeyRangeStatements")
keyrangeTransactions = stats.NewInt("UpdateStreamKeyRangeTransactions")
tablesStatements = stats.NewInt("UpdateStreamTablesStatements")
tablesTransactions = stats.NewInt("UpdateStreamTablesTransactions")
)
type UpdateStream struct {
mycnf *mysqlctl.Mycnf
actionLock sync.Mutex
state sync2.AtomicInt64
mysqld *mysqlctl.Mysqld
stateWaitGroup sync.WaitGroup
import (
"bytes"
"fmt"
"math/rand"
"strings"
"sync"
"time"
log "github.com/golang/glog"
"github.com/henryanand/vitess/go/stats"
"launchpad.net/gozk/zookeeper"
)
var (
cachedConnStates = stats.NewCounters("ZkCachedConn")
cachedConnStatesMutex sync.Mutex
)
func init() {
rand.Seed(time.Now().UnixNano())
}
/* When you need to talk to multiple zk cells, you need a simple
abstraction so you aren't caching clients all over the place.
ConnCache guarantees that you have at most one zookeeper connection per cell.
*/
const (
DISCONNECTED = 0
import (
"bytes"
"fmt"
"io"
log "github.com/golang/glog"
mproto "github.com/henryanand/vitess/go/mysql/proto"
"github.com/henryanand/vitess/go/stats"
"github.com/henryanand/vitess/go/sync2"
"github.com/henryanand/vitess/go/vt/binlog/proto"
"github.com/henryanand/vitess/go/vt/mysqlctl"
myproto "github.com/henryanand/vitess/go/vt/mysqlctl/proto"
)
var (
binlogStreamerErrors = stats.NewCounters("BinlogStreamerErrors")
ClientEOF = fmt.Errorf("binlog stream consumer ended the reply stream")
ServerEOF = fmt.Errorf("binlog stream connection was closed by mysqld")
// statementPrefixes are normal sql statement prefixes.
statementPrefixes = map[string]int{
"begin": proto.BL_BEGIN,
"commit": proto.BL_COMMIT,
"rollback": proto.BL_ROLLBACK,
"insert": proto.BL_DML,
"update": proto.BL_DML,
"delete": proto.BL_DML,
"create": proto.BL_DDL,
"alter": proto.BL_DDL,
"drop": proto.BL_DDL,
// NewQueryEngine creates a new QueryEngine.
// This is a singleton class.
// You must call this only once.
func NewQueryEngine(config Config) *QueryEngine {
qe := &QueryEngine{}
qe.schemaInfo = NewSchemaInfo(
config.QueryCacheSize,
time.Duration(config.SchemaReloadTime*1e9),
time.Duration(config.IdleTimeout*1e9),
)
qe.queryRuleInfo = NewQueryRuleInfo()
mysqlStats = stats.NewTimings("Mysql")
// Pools
qe.cachePool = NewCachePool(
"Rowcache",
config.RowCache,
time.Duration(config.QueryTimeout*1e9),
time.Duration(config.IdleTimeout*1e9),
)
qe.connPool = dbconnpool.NewConnectionPool(
"ConnPool",
config.PoolSize,
time.Duration(config.IdleTimeout*1e9),
)
qe.streamConnPool = dbconnpool.NewConnectionPool(
"StreamConnPool",
config.StreamPoolSize,
time.Duration(config.IdleTimeout*1e9),
)
// Services
qe.txPool = NewTxPool(
"TransactionPool",
config.TransactionCap,
time.Duration(config.TransactionTimeout*1e9),
time.Duration(config.TxPoolTimeout*1e9),
time.Duration(config.IdleTimeout*1e9),
)
qe.connKiller = NewConnectionKiller(1, time.Duration(config.IdleTimeout*1e9))
qe.consolidator = NewConsolidator()
qe.invalidator = NewRowcacheInvalidator(qe)
qe.streamQList = NewQueryList(qe.connKiller)
// Vars
qe.queryTimeout.Set(time.Duration(config.QueryTimeout * 1e9))
qe.spotCheckFreq = sync2.AtomicInt64(config.SpotCheckRatio * spotCheckMultiplier)
if config.StrictMode {
qe.strictMode.Set(1)
}
qe.strictTableAcl = config.StrictTableAcl
qe.maxResultSize = sync2.AtomicInt64(config.MaxResultSize)
qe.maxDMLRows = sync2.AtomicInt64(config.MaxDMLRows)
qe.streamBufferSize = sync2.AtomicInt64(config.StreamBufferSize)
// loggers
qe.accessCheckerLogger = logutil.NewThrottledLogger("accessChecker", 1*time.Second)
// Stats
stats.Publish("MaxResultSize", stats.IntFunc(qe.maxResultSize.Get))
stats.Publish("MaxDMLRows", stats.IntFunc(qe.maxDMLRows.Get))
stats.Publish("StreamBufferSize", stats.IntFunc(qe.streamBufferSize.Get))
stats.Publish("QueryTimeout", stats.DurationFunc(qe.queryTimeout.Get))
queryStats = stats.NewTimings("Queries")
QPSRates = stats.NewRates("QPS", queryStats, 15, 60*time.Second)
waitStats = stats.NewTimings("Waits")
killStats = stats.NewCounters("Kills")
infoErrors = stats.NewCounters("InfoErrors")
errorStats = stats.NewCounters("Errors")
internalErrors = stats.NewCounters("InternalErrors")
resultStats = stats.NewHistogram("Results", resultBuckets)
stats.Publish("RowcacheSpotCheckRatio", stats.FloatFunc(func() float64 {
return float64(qe.spotCheckFreq.Get()) / spotCheckMultiplier
}))
spotCheckCount = stats.NewInt("RowcacheSpotCheckCount")
return qe
}