//
//
// 等待Request请求的返回: Session最终被Block住
//
func (s *Session) handleResponse(r *Request) {
// 等待结果的出现
r.Wait.Wait()
// 将Err转换成为Exception
if r.Response.Err != nil {
r.Response.Data = GetThriftException(r, "proxy_session")
log.Infof(Magenta("---->Convert Error Back to Exception, Err: %v"), r.Response.Err)
}
// 如何处理Data和Err呢?
incrOpStats(r.Request.Name, microseconds()-r.Start)
}
func NewSessionSize(c thrift.TTransport, address string, verbose bool,
bufsize int, timeout int) *Session {
s := &Session{
CreateUnix: time.Now().Unix(),
RemoteAddress: address,
verbose: verbose,
TBufferedFramedTransport: NewTBufferedFramedTransport(c, time.Microsecond*100, 20),
}
// Reader 处理Client发送过来的消息
// Writer 将后端服务的数据返回给Client
log.Infof(Green("NewSession To: %s"), s.RemoteAddress)
return s
}
//
// 设置LogLevel
//
func SetLogLevel(level string) {
var lv = log.LEVEL_INFO
switch strings.ToLower(level) {
case "error":
lv = log.LEVEL_ERROR
case "warn", "warning":
lv = log.LEVEL_WARN
case "debug":
lv = log.LEVEL_DEBUG
case "info":
fallthrough
default:
lv = log.LEVEL_INFO
}
log.SetLevel(lv)
log.Infof("set log level to %s", lv)
}
func NewBackendConn(addr string, delegate *BackService, service string, verbose bool) *BackendConn {
requestMap, _ := NewRequestMap(4096)
var minSeqId int32
backendConnIndexMutex.Lock()
log.Infof("Create Backend Conn with index: %d", backendConnIndex)
// 主要是区分不同的: backendConnIndex
minSeqId = backendConnIndex
backendConnIndex += 1
if backendConnIndex > 100 {
backendConnIndex = 0
}
backendConnIndexMutex.Unlock()
// BACKEND_CONN_MAX_SEQ_ID = 1000000
minSeqId = minSeqId * BACKEND_CONN_MAX_SEQ_ID
bc := &BackendConn{
addr: addr,
service: service,
input: make(chan *Request, 1024),
seqNumRequestMap: requestMap,
currentSeqId: minSeqId,
minSeqId: minSeqId,
maxSeqId: minSeqId + BACKEND_CONN_MAX_SEQ_ID - 1,
Index: INVALID_ARRAY_INDEX,
delegate: delegate,
verbose: verbose,
}
go bc.Run()
return bc
}
func RpcMain(binaryName string, serviceDesc string, configCheck ConfigCheck,
serverFactory ServerFactorory, buildDate string, gitVersion string) {
// 1. 准备解析参数
usage = fmt.Sprintf(usage, binaryName, binaryName)
version := fmt.Sprintf("Version: %s\nBuildDate: %s\nDesc: %s\nAuthor: [email protected]", gitVersion, buildDate, serviceDesc)
args, err := docopt.Parse(usage, nil, true, version, true)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if s, ok := args["-V"].(bool); ok && s {
fmt.Println(Green(version))
os.Exit(1)
}
// 这就是为什么 Codis 傻乎乎起一个 http server的目的
if s, ok := args["--profile-addr"].(string); ok && len(s) > 0 {
go func() {
log.Printf(Red("Profile Address: %s"), s)
log.Println(http.ListenAndServe(s, nil))
}()
}
// 2. 解析Log相关的配置
log.SetLevel(log.LEVEL_INFO)
var maxKeepDays int = 3
if s, ok := args["--log-keep-days"].(string); ok && s != "" {
v, err := strconv.ParseInt(s, 10, 32)
if err != nil {
log.PanicErrorf(err, "invalid max log file keep days = %s", s)
}
maxKeepDays = int(v)
}
// set output log file
if s, ok := args["-L"].(string); ok && s != "" {
f, err := log.NewRollingFile(s, maxKeepDays)
if err != nil {
log.PanicErrorf(err, "open rolling log file failed: %s", s)
} else {
defer f.Close()
log.StdLog = log.New(f, "")
}
}
log.SetLevel(log.LEVEL_INFO)
log.SetFlags(log.Flags() | log.Lshortfile)
// set log level
if s, ok := args["--log-level"].(string); ok && s != "" {
SetLogLevel(s)
}
// 没有就没有
workDir, _ := args["--work-dir"].(string)
codeUrlVersion, _ := args["--code-url-version"].(string)
if len(workDir) == 0 {
workDir, _ = os.Getwd()
}
log.Printf("WorkDir: %s, CodeUrl: %s, Wd: %s", workDir, codeUrlVersion)
// 3. 解析Config
configFile := args["-c"].(string)
conf, err := LoadConf(configFile)
if err != nil {
log.PanicErrorf(err, "load config failed")
}
// 额外的配置信息
conf.WorkDir = workDir
conf.CodeUrlVersion = codeUrlVersion
if configCheck != nil {
configCheck(conf)
} else {
log.Panic("No Config Check Given")
}
// 每次启动的时候都打印版本信息
log.Infof(Green("-----------------\n%s\n--------------------------------------------------------------------"), version)
// 启动服务
server := serverFactory(conf)
server.Run()
}
func (s *NonBlockSession) Serve(d Dispatcher, maxPipeline int) {
var errlist errors.ErrorList
defer func() {
// 只限制第一个Error
if err := errlist.First(); err != nil {
log.Infof("Session [%p] closed, Error = %v", s, err)
} else {
log.Infof("Session [%p] closed, Quit", s)
}
}()
// 来自connection的各种请求
tasks := make(chan *Request, maxPipeline)
go func() {
defer func() {
// 出现错误了，直接关闭Session
s.Close()
log.Infof(Red("Session [%p] closed, Abandon %d Tasks"), s, len(tasks))
for _ = range tasks { // close(tasks)关闭for loop
}
}()
if err := s.loopWriter(tasks); err != nil {
errlist.PushBack(err)
}
}()
// 用于等待for中的go func执行完毕
var wait sync.WaitGroup
for true {
// Reader不停地解码， 将Request
request, err := s.ReadFrame()
if err != nil {
errlist.PushBack(err)
break
}
// 来自proxy的请求, request中不带有service
r, err1 := NewRequest(request, false)
if err1 != nil {
errlist.PushBack(err1)
break
}
wait.Add(1)
go func() {
// 异步执行(直接通过goroutine来调度，因为: SessionNonBlock中的不同的Request相互独立)
_ = s.handleRequest(r, d)
// if r.Request.TypeId != MESSAGE_TYPE_HEART_BEAT {
// log.Debugf(Magenta("[%p] --> SeqId: %d, Goroutine: %d"), s, r.Request.SeqId, runtime.NumGoroutine())
// }
// 数据请求完毕之后，将Request交给tasks, 然后再写回Client
tasks <- r
wait.Done()
// if r.Request.TypeId != MESSAGE_TYPE_HEART_BEAT {
// log.Printf("Time RT: %.3fms", float64(microseconds()-r.Start)*0.001)
// }
}()
}
// 等待go func执行完毕
wait.Wait()
close(tasks)
return
}
//
// 两参数是必须的: ProductName, zkAddress, frontAddr可以用来测试
//
func (p *ProxyServer) Run() {
var transport thrift.TServerTransport
var err error
log.Printf(Magenta("Start Proxy at Address: %s"), p.proxyAddr)
// 读取后端服务的配置
isUnixDomain := false
if !strings.Contains(p.proxyAddr, ":") {
if rpc_utils.FileExist(p.proxyAddr) {
os.Remove(p.proxyAddr)
}
transport, err = rpc_utils.NewTServerUnixDomain(p.proxyAddr)
isUnixDomain = true
} else {
transport, err = thrift.NewTServerSocket(p.proxyAddr)
}
if err != nil {
log.ErrorErrorf(err, "Server Socket Create Failed: %v, Front: %s", err, p.proxyAddr)
}
// 开始监听
// transport.Open()
transport.Listen()
ch := make(chan thrift.TTransport, 4096)
defer close(ch)
defer func() {
log.Infof(Red("==> Exit rpc_proxy"))
if err := recover(); err != nil {
log.Infof("Error rpc_proxy: %s", err)
}
}()
go func() {
var address string
for c := range ch {
// 为每个Connection建立一个Session
socket, ok := c.(rpc_utils.SocketAddr)
if isUnixDomain {
address = p.proxyAddr
} else if ok {
address = socket.Addr().String()
} else {
address = "unknow"
}
x := NewSession(c, address, p.verbose)
// Session独立处理自己的请求
go x.Serve(p.router, 1000)
}
}()
// Accept什么时候出错，出错之后如何处理呢?
for {
c, err := transport.Accept()
if err != nil {
log.ErrorErrorf(err, "Accept Error: %v", err)
break
} else {
ch <- c
}
}
}
// Session是同步处理请求，因此没有必要搞多个
func (s *Session) Serve(d Dispatcher, maxPipeline int) {
defer func() {
s.Close()
log.Infof(Red("==> Session Over: %s, Total %d Ops"), s.RemoteAddress, s.Ops)
if err := recover(); err != nil {
log.Infof(Red("Catch session error: %v"), err)
}
}()
for true {
// 1. 读取请求
request, err := s.ReadFrame()
s.Ops += 1
// 读取出错，直接退出
if err != nil {
err1, ok := err.(thrift.TTransportException)
if !ok || err1.TypeId() != thrift.END_OF_FILE {
log.ErrorErrorf(err, Red("ReadFrame Error: %v"), err)
}
// r.Recycle()
return
}
var r *Request
// 2. 处理请求
r, err = s.handleRequest(request, d)
if err != nil {
// r.Recycle() // 出错之后也要主动返回数据
log.ErrorErrorf(err, Red("handleRequest Error: %v"), err)
r.Response.Err = err
}
// 3. 等待请求处理完毕
s.handleResponse(r)
// 4. 将结果写回给Client
if s.verbose {
log.Debugf("[%s]Session#loopWriter --> client FrameSize: %d",
r.Service, len(r.Response.Data))
}
// 5. 将请求返回给Client, r.Response.Data ---> Client
_, err = s.TBufferedFramedTransport.Write(r.Response.Data)
r.Recycle() // 重置: Request
if err != nil {
log.ErrorErrorf(err, "Write back Data Error: %v", err)
return
}
// 6. Flush
err = s.TBufferedFramedTransport.FlushBuffer(true) // len(tasks) == 0
if err != nil {
log.ErrorErrorf(err, "Write back Data Error: %v", err)
return
}
// r.Recycle()
}
}
func (p *ThriftLoadBalanceServer) Run() {
// // 1. 创建到zk的连接
// 127.0.0.1:5555 --> 127_0_0_1:5555
exitSignal := make(chan os.Signal, 1)
signal.Notify(exitSignal, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
// syscall.SIGKILL
// kill -9 pid
// kill -s SIGKILL pid 还是留给运维吧
//
// 注册服务
evtExit := make(chan interface{})
// 初始状态为不上线
var state atomic2.Bool
state.Set(false)
stateChan := make(chan bool)
serviceEndpoint := RegisterService(p.serviceName, p.frontendAddr, p.lbServiceName,
p.topo, evtExit, p.config.WorkDir, p.config.CodeUrlVersion, &state, stateChan)
// var suideTime time.Time
// isAlive := true
// 3. 读取后端服务的配置
var transport thrift.TServerTransport
var err error
isUnixDomain := false
// 127.0.0.1:9999(以:区分不同的类型)
if !strings.Contains(p.frontendAddr, ":") {
if rpc_utils.FileExist(p.frontendAddr) {
os.Remove(p.frontendAddr)
}
transport, err = rpc_utils.NewTServerUnixDomain(p.frontendAddr)
isUnixDomain = true
} else {
transport, err = thrift.NewTServerSocket(p.frontendAddr)
}
if err != nil {
log.ErrorErrorf(err, "Server Socket Create Failed: %v", err)
panic(fmt.Sprintf("Invalid FrontendAddress: %s", p.frontendAddr))
}
err = transport.Listen()
if err != nil {
log.ErrorErrorf(err, "Server Socket Create Failed: %v", err)
panic(fmt.Sprintf("Binding Error FrontendAddress: %s", p.frontendAddr))
}
ch := make(chan thrift.TTransport, 4096)
defer close(ch)
// 等待后端服务起来
waitTicker := time.NewTicker(time.Second)
// 等待上线采用的策略:
// 1. 检测到有效的Worker注册之后，再等5s即可像zk注册; 避免了Worker没有连接上来，就有请求过来
// 2. 一旦注册之后，就不再使用该策略；避免服务故障时，lb频繁更新zk, 导致proxy等频繁读取zk
START_WAIT:
for true {
select {
case <-waitTicker.C:
if p.backendService.Active() <= 0 {
log.Infof("Sleep Waiting for back Service to Start")
time.Sleep(time.Second)
} else {
break START_WAIT
}
case <-exitSignal:
// 直接退出
transport.Interrupt()
transport.Close()
return
}
}
log.Infof("Stop Waiting")
// 停止: waitTicker, 再等等就继续了
waitTicker.Stop()
time.Sleep(time.Second * 5)
log.Infof("Begin to Reg To Zk...")
state.Set(true)
stateChan <- true
// 强制退出? TODO: Graceful退出
go func() {
<-exitSignal
// 通知RegisterService终止循环
evtExit <- true
log.Info(Green("Receive Exit Signals...."))
serviceEndpoint.DeleteServiceEndpoint(p.topo)
start := time.Now().Unix()
for true {
// 如果5s内没有接受到新的请求了，则退出
now := time.Now().Unix()
if now-p.lastRequestTime.Get() > 5 {
log.Printf(Red("[%s]Graceful Exit..."), p.serviceName)
break
} else {
log.Printf(Cyan("[%s]Sleeping %d seconds before Exit...\n"),
p.serviceName, now-start)
time.Sleep(time.Second)
}
}
transport.Interrupt()
transport.Close()
}()
go func() {
var address string
for c := range ch {
// 为每个Connection建立一个Session
socket, ok := c.(rpc_utils.SocketAddr)
if ok {
if isUnixDomain {
address = p.frontendAddr
} else {
address = socket.Addr().String()
}
} else {
address = "unknow"
}
x := NewNonBlockSession(c, address, p.verbose, &p.lastRequestTime)
// Session独立处理自己的请求
go x.Serve(p.backendService, 1000)
}
}()
// Accept什么时候出错，出错之后如何处理呢?
for {
c, err := transport.Accept()
if err != nil {
close(ch)
break
} else {
ch <- c
}
}
}