func (self *SchemaInfo) Reload() {
conn, err := self.ConnFactory()
if err != nil {
relog.Error("Could not get connection for reload: %v", err)
return
}
defer conn.Close()
query_for_schema_reload := fmt.Sprintf("show table status where unix_timestamp(create_time) > %v", self.LastReload.Unix())
self.LastReload = time.Now()
tables, err := conn.ExecuteFetch([]byte(query_for_schema_reload), 10000)
if err != nil {
relog.Error("Could not get table list for reload: %v", err)
return
}
if len(tables.Rows) == 0 {
return
}
for _, row := range tables.Rows {
tableName := row[0].(string)
relog.Info("Reloading: %s", tableName)
tableInfo := self.get(tableName)
if tableInfo != nil {
self.Put(tableInfo)
self.AlterTable(tableName, 0)
} else {
self.CreateTable(tableName, 0)
}
}
}
func (service *UmgmtService) gracefulShutdown() {
for e := service.shutdownCallbacks.Front(); e != nil; e = e.Next() {
if callback, ok := e.Value.(ShutdownCallback); ok {
callbackErr := callback()
if callbackErr != nil {
relog.Error("failed running shutdown callback:%v err:%v", callback, callbackErr)
}
} else {
relog.Error("bad callback %T %v", callback, callback)
}
}
service.done <- true
}
func (client *Client) GracefulShutdown() error {
request := new(Request)
reply := new(Reply)
err := client.Call("UmgmtService.GracefulShutdown", request, reply)
if err != nil {
relog.Error("rpc err: %v", err)
return err
}
if reply.ErrorCode != 0 {
relog.Error("GracefulShutdown err: %v %v", reply.ErrorCode, reply.Message)
return errors.New(reply.Message)
}
return nil
}
func (client *Client) CloseListeners() error {
request := new(Request)
reply := new(Reply)
err := client.Call("UmgmtService.CloseListeners", request, reply)
if err != nil {
relog.Error("rpc err: %v", err)
return err
}
if reply.ErrorCode != 0 {
relog.Error("CloseListeners err: %v %v", reply.ErrorCode, reply.Message)
return errors.New(reply.Message)
}
return nil
}
func (self *httpHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
conn := &httpConnectionBroker{c, req.Body}
codec := self.cFactory(conn)
if err := rpc.ServeRequest(codec); err != nil {
relog.Error("rpcwrap: %v", err)
}
}
func (server *UmgmtServer) Serve() error {
relog.Info("started umgmt server: %v", server.listener.Addr())
for !server.quit {
conn, err := server.listener.Accept()
if err != nil {
if checkError(err, syscall.EINVAL) {
if server.quit {
return nil
}
return err
}
// syscall.EMFILE, syscall.ENFILE could happen here if you run out of file descriptors
relog.Error("accept error %v", err)
continue
}
server.Lock()
server.connMap[conn] = true
server.Unlock()
rpc.ServeConn(conn)
server.Lock()
delete(server.connMap, conn)
server.Unlock()
}
return nil
}
func serve() {
AddShutdownCallback(ShutdownCallback(func() error { relog.Error("testserver GracefulShutdown callback"); return nil }))
err := ListenAndServe("/tmp/test-sock")
if err != nil {
relog.Fatal("listen err:%v", err)
}
relog.Info("test server finished")
}
func (self *rpcHandler) ServeHTTP(c http.ResponseWriter, req *http.Request) {
conn, _, err := c.(http.Hijacker).Hijack()
if err != nil {
relog.Error("rpc hijacking %s: %v", req.RemoteAddr, err)
return
}
io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n")
rpc.ServeCodec(self.cFactory(NewBufferedConnection(conn)))
}
func (c *conn) serve() {
defer func() {
err := recover()
if err == nil {
return
}
relog.Error("panic serving %v: %v\n", c.remoteAddr, err)
if c.rwc != nil {
c.rwc.Close()
}
c.close()
}()
for {
c.rwc.SetReadDeadline(time.Now().Add(agent_read_timeout))
c.rwc.SetWriteDeadline(time.Now().Add(agent_write_timeout))
req, err := readRequest(c.rw.Reader)
if err != nil {
msg := "CLIENT_ERROR"
if err == io.EOF {
break
} else if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
break
}
fmt.Fprintf(c.rwc, "%s %s\r\n", msg, err)
continue
}
err = c.handle_request(req)
if err != nil {
relog.Error("handle requesr error(%s)", err)
break
}
c.rw.Flush()
}
c.close()
}
func (self *ActivePool) kill(connid int64) {
self.Remove(connid)
killStats.Add("Queries", 1)
relog.Info("killing query %d", connid)
killConn := self.connPool.Get()
defer killConn.Recycle()
sql := []byte(fmt.Sprintf("kill %d", connid))
if _, err := killConn.ExecuteFetch(sql, 10000); err != nil {
relog.Error("Could not kill query %d: %v", connid, err)
}
}
func handleError(err *error) {
if x := recover(); x != nil {
terr := x.(*TabletError)
*err = terr
terr.RecordStats()
if terr.ErrorType == RETRY { // Retry errors are too spammy
return
}
relog.Error("%s", terr.Message)
}
}
// this is a callback to bind and startup an http server.
// usually it is called like:
// umgmt.AddStartupCallback(func () { umgmt.StartHttpServer(addr) })
func StartHttpServer(addr string) {
httpListener, httpErr := net.Listen("tcp", addr)
go func() {
if httpErr == nil {
httpListener = newHttpListener(httpListener)
AddListener(httpListener)
httpErr = http.Serve(httpListener, nil)
httpListener.Close()
}
if httpErr != nil {
switch e := httpErr.(type) {
case *net.OpError:
switch e.Err {
case syscall.EADDRINUSE:
relog.Fatal("StartHttpServer failed: %v", e)
}
case error:
// NOTE(msolomon) even though these are Errno objects, the constants
// are typed as os.Error.
switch e {
// FIXME(msolomon) this needs to be migrated into the system library
// because this needs to be properly handled in the accept loop.
case syscall.EMFILE, syscall.ENFILE:
relog.Error("non-fatal error serving HTTP: %s", e.Error())
case syscall.EINVAL:
// nothing - listener was probably closed
default:
relog.Error("http.ListenAndServe: " + httpErr.Error())
}
default:
relog.Error("http.ListenAndServe: " + httpErr.Error())
}
}
}()
}
func (service *UmgmtService) closeListeners() (err error) {
service.mutex.Lock()
defer service.mutex.Unlock()
for e := service.listeners.Front(); e != nil; e = e.Next() {
// NOTE(msolomon) we don't need the whole Listener interface, just Closer
//relog.Info("closeListeners %T %v", _listener, _listener)
if listener, ok := e.Value.(io.Closer); ok {
closeErr := listener.Close()
if closeErr != nil {
errMsg := fmt.Sprintf("failed to close listener:%v err:%v", listener, closeErr)
// just return that at least one error happened, the log will reveal the rest
err = errors.New(errMsg)
relog.Error("%s", errMsg)
}
// FIXME(msolomon) add a meaningful message telling what listener was closed
} else {
relog.Error("bad listener %T %v", listener, listener)
}
}
for _, f := range service.closeCallbacks {
go f()
}
return
}
func ListenAndServe(addr string) error {
rpc.Register(&defaultService)
DefaultServer = new(UmgmtServer)
DefaultServer.connMap = make(map[net.Conn]bool)
defer DefaultServer.Close()
var umgmtClient *Client
for i := 2; i > 0; i-- {
l, e := net.Listen("unix", addr)
if e != nil {
if checkError(e, syscall.EADDRINUSE) {
var clientErr error
umgmtClient, clientErr = Dial(addr)
if clientErr == nil {
closeErr := umgmtClient.CloseListeners()
if closeErr != nil {
relog.Error("closeErr:%v", closeErr)
}
// wait for rpc to finish
if rebindDelay > 0.0 {
relog.Info("delaying rebind: %vs", rebindDelay)
time.Sleep(rebindDelay)
}
continue
} else if checkError(clientErr, syscall.ECONNREFUSED) {
if unlinkErr := syscall.Unlink(addr); unlinkErr != nil {
relog.Error("can't unlink %v err:%v", addr, unlinkErr)
}
} else {
return e
}
} else {
return e
}
} else {
DefaultServer.listener = l
break
}
}
if DefaultServer.listener == nil {
panic("unable to rebind umgmt socket")
}
// register the umgmt server itself for dropping - this seems like
// the common case. i can't see when you *wouldn't* want to drop yourself
defaultService.addListener(DefaultServer)
defaultService.addShutdownCallback(func() error {
return DefaultServer.handleGracefulShutdown()
})
// fire off the startup callbacks. if these bind ports, they should
// call AddListener.
for e := defaultService.startupCallbacks.Front(); e != nil; e = e.Next() {
if startupCallback, ok := e.Value.(StartupCallback); ok {
startupCallback()
} else {
relog.Error("bad callback %T %v", e.Value, e.Value)
}
}
if umgmtClient != nil {
go func() {
time.Sleep(lameDuckPeriod)
umgmtClient.GracefulShutdown()
umgmtClient.Close()
}()
}
return DefaultServer.Serve()
}
func main() {
memProfileRate := flag.Int("mem-profile-rate", 512*1024, "profile every n bytes allocated")
maxOpenFds := flag.Uint64("max-open-fds", 32768, "max open file descriptors")
configFile := flag.String("config", "", "config file name")
dbConfigFile := flag.String("dbconfig", "", "db config file name")
lameDuckPeriod := flag.Float64("lame-duck-period", DefaultLameDuckPeriod,
"how long to give in-flight transactions to finish")
rebindDelay := flag.Float64("rebind-delay", DefaultRebindDelay,
"artificial delay before rebinding a hijacked listener")
logfileName := flag.String("logfile", "/dev/stderr", "base log file name")
logFrequency := flag.Int64("logfile.frequency", 0, "rotation frequency in seconds")
logMaxSize := flag.Int64("logfile.maxsize", 0, "max file size in bytes")
logMaxFiles := flag.Int64("logfile.maxfiles", 0, "max number of log files")
queryLog := flag.String("querylog", "", "for testing: log all queries to this file")
flag.Parse()
exportBinaryVersion()
runtime.MemProfileRate = *memProfileRate
f, err := logfile.Open(*logfileName, *logFrequency, *logMaxSize, *logMaxFiles)
if err != nil {
panic(fmt.Sprintf("unable to open logfile %s", *logfileName))
}
logger := relog.New(f, "vtocc ",
log.Ldate|log.Lmicroseconds|log.Lshortfile, relog.DEBUG)
relog.SetLogger(logger)
if *queryLog != "" {
if f, err = os.OpenFile(*queryLog, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err == nil {
ts.QueryLogger = relog.New(f, "", log.Ldate|log.Lmicroseconds, relog.DEBUG)
}
}
unmarshalFile(*configFile, &config)
unmarshalFile(*dbConfigFile, &dbconfig)
// work-around for jsonism
if v, ok := dbconfig["port"].(float64); ok {
dbconfig["port"] = int(v)
}
fdLimit := &syscall.Rlimit{*maxOpenFds, *maxOpenFds}
if err = syscall.Setrlimit(RLIMIT_NOFILE, fdLimit); err != nil {
relog.Fatal("can't Setrlimit %#v: err %v", *fdLimit, err)
} else {
relog.Info("set max-open-fds = %v", *maxOpenFds)
}
snitch.RegisterCommand("reload_schema", "Rescan the schema for new tables", ReloadHandler)
snitch.Register()
qm := &OccManager{config, dbconfig}
rpc.Register(qm)
ts.StartQueryService(
config.PoolSize,
config.TransactionCap,
config.TransactionTimeout,
config.MaxResultSize,
config.QueryCacheSize,
config.SchemaReloadTime,
config.QueryTimeout,
config.IdleTimeout,
)
ts.AllowQueries(ts.GenericConnectionCreator(dbconfig), nil)
rpc.HandleHTTP()
jsonrpc.ServeHTTP()
jsonrpc.ServeRPC()
bsonrpc.ServeHTTP()
bsonrpc.ServeRPC()
relog.Info("started vtocc %v", config.Port)
// we delegate out startup to the micromanagement server so these actions
// will occur after we have obtained our socket.
usefulLameDuckPeriod := float64(config.QueryTimeout + 1)
if usefulLameDuckPeriod > *lameDuckPeriod {
*lameDuckPeriod = usefulLameDuckPeriod
relog.Info("readjusted -lame-duck-period to %f", *lameDuckPeriod)
}
umgmt.SetLameDuckPeriod(float32(*lameDuckPeriod))
umgmt.SetRebindDelay(float32(*rebindDelay))
umgmt.AddStartupCallback(func() {
umgmt.StartHttpServer(fmt.Sprintf(":%v", config.Port))
})
umgmt.AddStartupCallback(func() {
sighandler.SetSignalHandler(syscall.SIGTERM,
umgmt.SigTermHandler)
})
umgmt.AddCloseCallback(func() {
ts.DisallowQueries()
})
umgmt.AddShutdownCallback(func() error {
HandleGracefulShutdown()
return nil
})
umgmtSocket := fmt.Sprintf(config.UmgmtSocket, config.Port)
if umgmtErr := umgmt.ListenAndServe(umgmtSocket); umgmtErr != nil {
relog.Error("umgmt.ListenAndServe err: %v", umgmtErr)
}
relog.Info("done")
}
func (self *SqlQuery) Execute(query *Query, reply *QueryResult) (err error) {
defer func() {
if x := recover(); x != nil {
terr := x.(*TabletError)
err = terr
terr.RecordStats()
if terr.ErrorType == RETRY || terr.SqlError == DUPLICATE_KEY { // suppress these errors in logs
return
}
relog.Error("%s: %v", terr.Message, query)
}
}()
// allow shutdown state if we're in a transaction
allowShutdown := (query.TransactionId != 0)
self.checkState(query.SessionId, allowShutdown)
self.mu.RLock()
defer self.mu.RUnlock()
if query.BindVariables == nil { // will help us avoid repeated nil checks
query.BindVariables = make(map[string]interface{})
}
// cheap hack: strip trailing comment into a special bind var
stripTrailing(query)
basePlan, tableInfo := self.schemaInfo.GetPlan(query.Sql, len(query.BindVariables) != 0)
defer self.schemaInfo.Put(tableInfo)
plan := &CompiledPlan{basePlan, tableInfo, query.BindVariables, query.TransactionId, query.ConnectionId}
// Need upfront connection for DMLs and transactions
if query.TransactionId != 0 {
conn := self.activeTxPool.Get(query.TransactionId)
defer conn.Recycle()
var invalidator CacheInvalidator
if tableInfo != nil && tableInfo.CacheType != 0 {
invalidator = conn.DirtyKeys(plan.TableName)
}
switch plan.PlanId {
case sqlparser.PLAN_PASS_DML:
defer queryStats.Record("PASS_DML", time.Now())
*reply = *self.directFetch(conn, plan.FullQuery, plan.BindVars, nil, nil)
case sqlparser.PLAN_INSERT_PK:
defer queryStats.Record("PLAN_INSERT_PK", time.Now())
*reply = *self.execInsertPK(conn, plan, invalidator)
case sqlparser.PLAN_INSERT_SUBQUERY:
defer queryStats.Record("PLAN_INSERT_SUBQUERY", time.Now())
*reply = *self.execInsertSubquery(conn, plan, invalidator)
case sqlparser.PLAN_DML_PK:
defer queryStats.Record("DML_PK", time.Now())
*reply = *self.execDMLPK(conn, plan, invalidator)
case sqlparser.PLAN_DML_SUBQUERY:
defer queryStats.Record("DML_SUBQUERY", time.Now())
*reply = *self.execDMLSubquery(conn, plan, invalidator)
default: // select or set in a transaction, just count as select
defer queryStats.Record("PASS_SELECT", time.Now())
*reply = *self.directFetch(conn, plan.FullQuery, plan.BindVars, nil, nil)
}
} else {
switch plan.PlanId {
case sqlparser.PLAN_PASS_SELECT:
if plan.Reason == sqlparser.REASON_FOR_UPDATE {
panic(NewTabletError(FAIL, "Disallowed outside transaction: %s", query.Sql))
}
defer queryStats.Record("PASS_SELECT", time.Now())
*reply = *self.qFetch(plan, plan.FullQuery, nil)
case sqlparser.PLAN_SELECT_PK:
defer queryStats.Record("SELECT_PK", time.Now())
*reply = *self.execPK(plan)
case sqlparser.PLAN_SELECT_SUBQUERY:
defer queryStats.Record("SELECT_SUBQUERY", time.Now())
*reply = *self.execSubquery(plan)
case sqlparser.PLAN_SELECT_CACHE_RESULT:
defer queryStats.Record("SELECT_CACHE_RESULT", time.Now())
*reply = *self.execCacheResult(plan)
case sqlparser.PLAN_SET:
defer queryStats.Record("SET", time.Now())
*reply = *self.execSet(plan)
default:
panic(NewTabletError(FAIL, "DMLs not allowed outside of transactions"))
}
}
if plan.PlanId.IsSelect() {
resultStats.Add(int64(reply.RowsAffected))
}
return nil
}
func logError() {
if x := recover(); x != nil {
relog.Error("%s", x.(*TabletError).Message)
}
}