//
// 将消息发送到Backend上去
//
func (s *BackService) HandleRequest(req *Request) (err error) {
// 并发度可能很高
backendConn := s.NextBackendConn()
s.lastRequestTime.Set(time.Now().Unix())
if backendConn == nil {
// 没有后端服务
if s.verbose {
log.Println(Red("No BackSocket Found for service:"), s.serviceName)
}
// 从errMsg来构建异常
errMsg := GetWorkerNotFoundData(req, "BackService")
req.Response.Data = errMsg
// XXX: 没有等待，req.Wait.Wait() 直接返回
return nil
} else {
if s.verbose {
log.Println("SendMessage With: ", backendConn.Addr(), "For Service: ", s.serviceName)
}
backendConn.PushBack(req)
return nil
}
}
// Router负责监听zk中服务列表的变化
func (bk *Router) WatchServices() {
var evtbus chan interface{} = make(chan interface{}, 2)
// 1. 保证Service目录存在，否则会报错
servicesPath := bk.topo.ProductServicesPath()
_, e1 := bk.topo.CreateDir(servicesPath)
if e1 != nil {
log.PanicErrorf(e1, "Zk Path Create Failed: %s", servicesPath)
}
go func() {
for true {
// 无限监听
services, err := bk.topo.WatchChildren(servicesPath, evtbus)
if err == nil {
bk.serviceLock.Lock()
// 保证数据更新是有效的
oldServices := bk.services
bk.services = make(map[string]*BackService, len(services))
for _, service := range services {
log.Println("Found Service: ", Magenta(service))
back, ok := oldServices[service]
if ok {
bk.services[service] = back
delete(oldServices, service)
} else {
bk.addBackService(service)
}
}
if len(oldServices) > 0 {
for _, conn := range oldServices {
// 标记下线(现在应该不会有新的请求，最多只会处理一些收尾的工作
conn.Stop()
}
}
bk.serviceLock.Unlock()
// 等待事件
<-evtbus
} else {
log.ErrorErrorf(err, "zk watch error: %s, error: %v\n",
servicesPath, err)
time.Sleep(time.Duration(5) * time.Second)
}
}
}()
// 读取zk, 等待
log.Println("ProductName: ", Magenta(bk.topo.ProductName))
}
//
// 删除Service Endpoint
//
func (s *ServiceEndpoint) DeleteServiceEndpoint(top *Topology) {
path := top.ProductServiceEndPointPath(s.Service, s.ServiceId)
if ok, _ := top.Exist(path); ok {
zkhelper.DeleteRecursive(top.ZkConn, path, -1)
log.Println(Red("DeleteServiceEndpoint"), "Path: ", path)
}
}
// 创建指定的Path
func (top *Topology) CreateDir(path string) (string, error) {
dir := top.FullPath(path)
if ok, _ := top.Exist(dir); ok {
log.Println("Path Exists")
return dir, nil
} else {
return zkhelper.CreateRecursive(top.ZkConn, dir, "", 0, zkhelper.DefaultDirACLs())
}
}
// 获取可以直接返回给Client的response
func (s *NonBlockSession) handleResponse(r *Request) {
// 将Err转换成为Exception
if r.Response.Err != nil {
log.Println("#handleResponse, Error ----> Reponse Data")
r.Response.Data = GetThriftException(r, "nonblock_session")
}
incrOpStats(r.Request.Name, microseconds()-r.Start)
}
//
// 设置RPC Proxy的数据:
// 绑定的前端的ip/port, 例如: {"rpc_front": "tcp://127.0.0.1:5550"}
//
func (top *Topology) SetRpcProxyData(proxyInfo map[string]interface{}) error {
path := top.FullPath("/rpc_proxy")
data, err := json.Marshal(proxyInfo)
if err != nil {
return err
}
// topo.FlagEphemeral 这里的ProxyInfo是手动配置的，需要持久化
path, err = CreateOrUpdate(top.ZkConn, path, string(data), 0, zkhelper.DefaultDirACLs(), true)
log.Println(green("SetRpcProxyData"), "Path: ", path, ", Error: ", err, ", Data: ", string(data))
return err
}
//
// 读取RPC Proxy的数据:
// 绑定的前端的ip/port, 例如: {"rpc_front": "tcp://127.0.0.1:5550"}
//
func (top *Topology) GetRpcProxyData() (proxyInfo map[string]interface{}, e error) {
path := top.FullPath("/rpc_proxy")
data, _, err := top.ZkConn.Get(path)
log.Println("Data: ", data, ", err: ", err)
if err != nil {
return nil, err
}
proxyInfo = make(map[string]interface{})
err = json.Unmarshal(data, &proxyInfo)
if err != nil {
return nil, err
} else {
return proxyInfo, nil
}
}
//
// 通过参数依赖，保证getFrontendAddr的调用位置（必须等待Host, IpPrefix, Port读取完毕之后)
//
func (conf *Config) getFrontendAddr(frontHost, ipPrefix, frontPort string) string {
if conf.FrontSock != "" {
return conf.FrontSock
}
var frontendAddr = ""
// 如果没有指定FrontHost, 则自动根据 IpPrefix来进行筛选，
// 例如: IpPrefix: 10., 那么最终内网IP： 10.4.10.2之类的被选中
if frontHost == "" {
log.Println("FrontHost: ", frontHost, ", Prefix: ", ipPrefix)
if ipPrefix != "" {
frontHost = GetIpWithPrefix(ipPrefix)
}
}
if frontPort != "" && frontHost != "" {
frontendAddr = fmt.Sprintf("%s:%s", frontHost, frontPort)
}
return frontendAddr
}
//
// 注册一个服务的Endpoints
//
func (s *ServiceEndpoint) AddServiceEndpoint(topo *Topology) error {
path := topo.ProductServiceEndPointPath(s.Service, s.ServiceId)
data, err := json.Marshal(s)
if err != nil {
return err
}
// 创建Service(XXX: Service本身不包含数据)
CreateRecursive(topo.ZkConn, os_path.Dir(path), "", 0, zkhelper.DefaultDirACLs())
// 当前的Session挂了，服务就下线
// topo.FlagEphemeral
// 参考： https://www.box.com/blog/a-gotcha-when-using-zookeeper-ephemeral-nodes/
// 如果之前的Session信息还存在，则先删除；然后再添加
topo.ZkConn.Delete(path, -1)
var pathCreated string
pathCreated, err = topo.ZkConn.Create(path, []byte(data), int32(zookeeper.FlagEphemeral), zkhelper.DefaultFileACLs())
log.Println(Green("AddServiceEndpoint"), "Path: ", pathCreated, ", Error: ", err)
return err
}
func RpcMain(binaryName string, serviceDesc string, configCheck ConfigCheck,
serverFactory ServerFactorory, buildDate string, gitVersion string) {
// 1. 准备解析参数
usage = fmt.Sprintf(usage, binaryName, binaryName)
version := fmt.Sprintf("Version: %s\nBuildDate: %s\nDesc: %s\nAuthor: [email protected]", gitVersion, buildDate, serviceDesc)
args, err := docopt.Parse(usage, nil, true, version, true)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if s, ok := args["-V"].(bool); ok && s {
fmt.Println(Green(version))
os.Exit(1)
}
// 这就是为什么 Codis 傻乎乎起一个 http server的目的
if s, ok := args["--profile-addr"].(string); ok && len(s) > 0 {
go func() {
log.Printf(Red("Profile Address: %s"), s)
log.Println(http.ListenAndServe(s, nil))
}()
}
// 2. 解析Log相关的配置
log.SetLevel(log.LEVEL_INFO)
var maxKeepDays int = 3
if s, ok := args["--log-keep-days"].(string); ok && s != "" {
v, err := strconv.ParseInt(s, 10, 32)
if err != nil {
log.PanicErrorf(err, "invalid max log file keep days = %s", s)
}
maxKeepDays = int(v)
}
// set output log file
if s, ok := args["-L"].(string); ok && s != "" {
f, err := log.NewRollingFile(s, maxKeepDays)
if err != nil {
log.PanicErrorf(err, "open rolling log file failed: %s", s)
} else {
defer f.Close()
log.StdLog = log.New(f, "")
}
}
log.SetLevel(log.LEVEL_INFO)
log.SetFlags(log.Flags() | log.Lshortfile)
// set log level
if s, ok := args["--log-level"].(string); ok && s != "" {
SetLogLevel(s)
}
// 没有就没有
workDir, _ := args["--work-dir"].(string)
codeUrlVersion, _ := args["--code-url-version"].(string)
if len(workDir) == 0 {
workDir, _ = os.Getwd()
}
log.Printf("WorkDir: %s, CodeUrl: %s, Wd: %s", workDir, codeUrlVersion)
// 3. 解析Config
configFile := args["-c"].(string)
conf, err := LoadConf(configFile)
if err != nil {
log.PanicErrorf(err, "load config failed")
}
// 额外的配置信息
conf.WorkDir = workDir
conf.CodeUrlVersion = codeUrlVersion
if configCheck != nil {
configCheck(conf)
} else {
log.Panic("No Config Check Given")
}
// 每次启动的时候都打印版本信息
log.Infof(Green("-----------------\n%s\n--------------------------------------------------------------------"), version)
// 启动服务
server := serverFactory(conf)
server.Run()
}
//
// go test proxy -v -run "TestBackend"
//
func TestBackend(t *testing.T) {
// 作为一个Server
transport, err := thrift.NewTServerSocket("127.0.0.1:0")
assert.NoError(t, err)
err = transport.Open() // 打开Transport
assert.NoError(t, err)
defer transport.Close()
err = transport.Listen() // 开始监听
assert.NoError(t, err)
addr := transport.Addr().String()
fmt.Println("Addr: ", addr)
var requestNum int32 = 10
requests := make([]*Request, 0, requestNum)
var i int32
for i = 0; i < requestNum; i++ {
buf := make([]byte, 100, 100)
l := fakeData("Hello", thrift.CALL, i+1, buf[0:0])
buf = buf[0:l]
req, _ := NewRequest(buf, false)
req.Wait.Add(1) // 因为go routine可能还没有执行，代码就跑到最后面进行校验了
assert.Equal(t, i+1, req.Request.SeqId, "Request SeqId是否靠谱")
requests = append(requests, req)
}
go func() {
// 客户端代码
bc := NewBackendConn(addr, nil, "test", true)
bc.currentSeqId = 10
// 准备发送数据
var i int32
for i = 0; i < requestNum; i++ {
fmt.Println("Sending Request to Backend Conn", i)
bc.PushBack(requests[i])
requests[i].Wait.Done()
}
// 需要等待数据返回?
time.Sleep(time.Second * 2)
}()
go func() {
// 服务器端代码
tran, err := transport.Accept()
if err != nil {
log.ErrorErrorf(err, "Error: %v\n", err)
}
assert.NoError(t, err)
bt := NewTBufferedFramedTransport(tran, time.Microsecond*100, 2)
// 在当前的这个t上读写数据
var i int32
for i = 0; i < requestNum; i++ {
request, err := bt.ReadFrame()
assert.NoError(t, err)
req, _ := NewRequest(request, false)
assert.Equal(t, req.Request.SeqId, i+10)
fmt.Printf("Server Got Request, and SeqNum OK, Id: %d, Frame Size: %d\n", i, len(request))
// 回写数据
bt.Write(request)
bt.FlushBuffer(true)
}
tran.Close()
}()
fmt.Println("Requests Len: ", len(requests))
for idx, r := range requests {
r.Wait.Wait()
// r 原始的请求
req, _ := NewRequest(r.Response.Data, false)
log.Printf(Green("SeqMatch[%d]: Orig: %d, Return: %d\n"), idx, req.Request.SeqId, r.Request.SeqId)
assert.Equal(t, req.Request.SeqId, r.Request.SeqId)
}
log.Println("OK")
}