// PIPELINE #2: agent
// all the packets from handleClient() will be handled
func agent(sess *Session, in chan []byte, out *Buffer) {
defer wg.Done() // will decrease waitgroup by one, useful for manual server shutdown
defer utils.PrintPanicStack()
// init session
sess.MQ = make(chan pb.Game_Frame, DEFAULT_MQ_SIZE)
sess.ConnectTime = time.Now()
sess.LastPacketTime = time.Now()
// minute timer
min_timer := time.After(time.Minute)
// cleanup work
defer func() {
close(sess.Die)
if sess.Stream != nil {
sess.Stream.CloseSend()
}
}()
// >> the main message loop <<
// handles 4 types of message:
// 1. from client
// 2. from game service
// 3. timer
// 4. server shutdown signal
for {
select {
case msg, ok := <-in: // packet from network
if !ok {
return
}
sess.PacketCount++
sess.PacketTime = time.Now()
if result := proxy_user_request(sess, msg); result != nil {
out.send(sess, result)
}
sess.LastPacketTime = sess.PacketTime
case frame := <-sess.MQ: // packets from game
switch frame.Type {
case pb.Game_Message:
out.send(sess, frame.Message)
case pb.Game_Kick:
sess.Flag |= SESS_KICKED_OUT
}
case <-min_timer: // minutes timer
timer_work(sess, out)
min_timer = time.After(time.Minute)
case <-die: // server is shuting down...
sess.Flag |= SESS_KICKED_OUT
}
// see if the player should be kicked out.
if sess.Flag&SESS_KICKED_OUT != 0 {
return
}
}
}
// agent of user
func agent(sess *Session, in chan []byte, out *Buffer, sess_die chan bool) {
defer wg.Done()
defer utils.PrintPanicStack()
// init session
sess.MQ = make(chan spp.Game_Frame, DEFAULT_MQ_SIZE)
sess.ConnectTime = time.Now()
sess.LastPacketTime = time.Now()
// minute timer
min_timer := time.After(time.Minute)
// cleanup work
defer func() {
close(sess_die)
if sess.Stream != nil {
sess.Stream.CloseSend()
}
}()
// >> the main message loop <<
for {
select {
case msg, ok := <-in: // packet from network
if !ok {
return
}
sess.PacketCount++
sess.PacketTime = time.Now()
if result := proxy_user_request(sess, msg); result != nil {
out.send(sess, result)
}
sess.LastPacketTime = sess.PacketTime
case frame := <-sess.MQ:
switch frame.Type {
case spp.Game_Message:
out.send(sess, frame.Message)
case spp.Game_Kick:
sess.Flag |= SESS_KICKED_OUT
}
case <-min_timer: // minutes timer
timer_work(sess, out)
min_timer = time.After(time.Minute)
case <-die: // server is shuting down...
sess.Flag |= SESS_KICKED_OUT
}
// see if the player should be kicked out.
if sess.Flag&SESS_KICKED_OUT != 0 {
return
}
}
}
// packet sending goroutine
func (buf *Buffer) start() {
defer utils.PrintPanicStack()
for {
select {
case data := <-buf.pending:
buf.raw_send(data)
case <-buf.ctrl: // receive session end signal
close(buf.pending)
// close the connection
buf.conn.Close()
return
}
}
}
func main() {
// to catch all uncaught panic
defer utils.PrintPanicStack()
// open profiling
go func() {
log.Info(http.ListenAndServe("0.0.0.0:6060", nil))
}()
// set log prefix
log.SetPrefix(SERVICE)
// resolve address & start listening
tcpAddr, err := net.ResolveTCPAddr("tcp4", _port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
log.Info("listening on:", listener.Addr())
// startup
startup()
LOOP:
// loop accepting
for {
conn, err := listener.AcceptTCP()
if err != nil {
log.Warning("accept failed:", err)
continue
}
go handleClient(conn) // start a goroutine for every incoming connection for reading
// check server close signal
select {
case <-die:
listener.Close()
break LOOP
default:
}
}
// server closed, wait forever
// other options:
// select{} -- may cause deadlock detected error, not tested yet
for {
<-time.After(time.Second)
}
}
func main() {
defer utils.PrintPanicStack()
go func() {
log.Info(http.ListenAndServe("0.0.0.0:6060", nil))
}()
log.SetPrefix(SERVICE)
// resolve
tcpAddr, err := net.ResolveTCPAddr("tcp4", _port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
log.Info("listening on:", listener.Addr())
// init services
sp.Init()
// startup
startup()
// loop accepting
LOOP:
for {
conn, err := listener.AcceptTCP()
if err != nil {
log.Warning("accept failed:", err)
continue
}
go handleClient(conn)
// check server close signal
select {
case <-die:
listener.Close()
break LOOP
default:
}
}
// server closed, wait forever
for {
<-time.After(time.Second)
}
}
// handle unix signals
func sig_handler() {
defer utils.PrintPanicStack()
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGTERM)
for {
msg := <-ch
switch msg {
case syscall.SIGTERM: // 关闭agent
close(die)
log.Info("sigterm received")
log.Info("waiting for agents close, please wait...")
wg.Wait()
log.Info("agent shutdown.")
os.Exit(0)
}
}
}
// packet sending goroutine
// 数据包发送协程
func (buf *Buffer) start() {
defer utils.PrintPanicStack()
for {
select {
//监听数据包管道
case data := <-buf.pending:
buf.raw_send(data)
//监听关闭管道
//?Important
//ctrl 指向的是 sess_die, 在main.go里面,已经有select监听了? 此处会不会造成问题
case <-buf.ctrl: //receive session end signal
//关闭接收管道
close(buf.pending)
//关闭conn连接
buf.conn.Close()
return
}
}
}
//游戏入口
func main() {
defer utils.PrintPanicStack()
go func() {
log.Info(http.ListenAndServe("0.0.0.0:6060", nil))
}()
log.SetPrefix(SERVICE)
tcpAddr, err := net.ResolveTCPAddr("tcp4", _port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
log.Info("listening on:", listener.Addr())
// loop accepting
for {
conn, err := listener.AcceptTCP()
if err != nil {
log.Warning("accept failed:", err)
continue
}
go handleClient(conn)
// check server close signal
select {
case <-die:
listener.Close()
goto FINAL
default:
}
}
FINAL:
// server closed, wait forever
// 此处为什么要等1秒
// 此处等待1秒钟，只是为了wg.Wait() 可以执行到，从而阻塞主线程
for {
<-time.After(time.Second)
}
}
// client protocol handle proxy
func proxy_user_request(sess *Session, p []byte) []byte {
start := time.Now()
defer utils.PrintPanicStack()
//解密
if sess.Flag&SESS_ENCRYPT != 0 {
//使用2组密钥匙, 增加破解难度
sess.Decoder.XORKeyStream(p, p)
}
//封装成reader
reader := packet.Reader(p)
// 读客户端数据包序列号(1,2,3...)
// 可避免重放攻击-REPLAY-ATTACK
//数据包的前4个字节存放的是客户端的发包数量
//客户端每次发包要包含第几次发包信息
seq_id, err := reader.ReadU32()
if err != nil {
log.Error("read client timestamp failed:", err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 数据包个数验证
if seq_id != sess.PacketCount {
//数据包真实长度是，总长度-4个字节的包个数长度-2个字节的协议号长度
log.Errorf("illegal packet sequeue id:%v should be:%v size:%v", seq_id, sess.PacketCount, len(p)-6)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 读协议号
b, err := reader.ReadS16()
if err != nil {
log.Error("read protocol number failed.")
sess.Flag |= SESS_KICKED_OUT
return nil
}
//根据协议号段 做服务划分
var ret []byte
if b > MAX_PROTO_NUM {
//去除4字节发包个数,将其余的向前传递
if err := forward(sess, p[4:]); err != nil {
log.Errorf("service id:%v execute failed, error:%v", b, err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
} else {
if h := client_handler.Handlers[b]; h != nil {
ret = h(sess, reader)
} else {
log.Errorf("service id:%v not bind", b)
sess.Flag |= SESS_KICKED_OUT
return nil
}
}
// 统计处理时间
elasped := time.Now().Sub(start)
if b != 0 { //排除心跳包日志
log.Trace("[REQ]", client_handler.RCode[b])
_statter.Timing(1.0, fmt.Sprintf("%v%v", STATSD_PREFIX, client_handler.RCode[b]), elasped)
}
return ret
}
// start a goroutine when a new connection is accepted
func handleClient(conn *net.TCPConn) {
defer utils.PrintPanicStack()
// set per-connection socket buffer
conn.SetReadBuffer(SO_RCVBUF)
// set initial socket buffer
conn.SetWriteBuffer(SO_SNDBUF)
// initial network control struct
header := make([]byte, 2)
in := make(chan []byte)
defer func() {
close(in) // session will close
}()
// create a new session object for the connection
var sess Session
host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
log.Error("cannot get remote address:", err)
return
}
sess.IP = net.ParseIP(host)
log.Infof("new connection from:%v port:%v", host, port)
// session die signal
sess.Die = make(chan struct{})
// create a write buffer
out := new_buffer(conn, sess.Die)
go out.start()
// start one agent for handling packet
wg.Add(1)
go agent(&sess, in, out)
// network loop
for {
// solve dead link problem
conn.SetReadDeadline(time.Now().Add(TCP_READ_DEADLINE * time.Second))
n, err := io.ReadFull(conn, header)
if err != nil {
log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
size := binary.BigEndian.Uint16(header)
// alloc a byte slice for reading
payload := make([]byte, size)
// read msg
n, err = io.ReadFull(conn, payload)
if err != nil {
log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
select {
case in <- payload: // payload queued
case <-sess.Die:
log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
return
}
}
}
// client protocol handle proxy
func proxy_user_request(sess *Session, p []byte) []byte {
start := time.Now()
defer utils.PrintPanicStack(sess, p)
// 解密
if sess.Flag&SESS_ENCRYPT != 0 {
sess.Decoder.Codec(p)
}
// 封装为reader
reader := packet.Reader(p)
// 读客户端数据包序列号(1,2,3...)
// 可避免重放攻击-REPLAY-ATTACK
seq_id, err := reader.ReadU32()
if err != nil {
log.Error("read client timestamp failed:", err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 读协议号
b, err := reader.ReadS16()
if err != nil {
log.Error("read protocol number failed.")
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 数据包序列号验证
if seq_id != sess.PacketCount {
log.Errorf("illegal packet sequence id:%v should be:%v proto:%v size:%v", seq_id, sess.PacketCount, b, len(p)-6)
sess.Flag |= SESS_KICKED_OUT
return nil
}
var ret []byte
if b > MAX_PROTO_NUM { // game协议
// 透传
ret, err = forward(sess, p)
if err != nil {
log.Errorf("service id:%v execute failed", b)
sess.Flag |= SESS_KICKED_OUT
return nil
}
} else { // agent保留协议段 [0, MAX_PROTO_NUM]
// handle有效性检查
h := client_handler.Handlers[b]
if h == nil {
log.Errorf("service id:%v not bind", b)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 执行
ret = h(sess, reader)
}
// 统计处理时间
elasped := time.Now().Sub(start)
if b != 0 { // 排除心跳包日志
log.Trace("[REQ]", b)
_statter.Timing(1.0, fmt.Sprintf("%v%v", STATSD_PREFIX, b), elasped)
}
return ret
}
// start a goroutine when a new connection is accepted
func handleClient(conn *net.TCPConn) {
defer utils.PrintPanicStack()
// set per-connection socket buffer
//设置conn连接接受缓存的大小, 当超过缓存时, 会进入阻塞状态,等待被读取
conn.SetReadBuffer(SO_RCVBUF)
// set initial socket buffer
//设置conn连接发送缓存的大小, 当超过缓存时, 会进入阻塞状态,等待被发送成功
conn.SetWriteBuffer(SO_SNDBUF)
// initial network control struct
// 初始化2个字节数组, 用于存储header长度, 既后面要读取的文件长度
header := make([]byte, 2)
// 输入流通道, 解析后的数据将放入，等待被处理
in := make(chan []byte)
//设置延迟函数，当玩家断开连接时, 函数退出之前，关闭输入流
defer func() {
close(in) // session will close
}()
// create a new session object for the connection
//创建session对象, 用于封装客户端和服务器的信息交换
var sess Session
host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
log.Error("cannot get remote address:", err)
return
}
//存储用户ip
sess.IP = net.ParseIP(host)
//打印用户的ip和端口, 用户可能会双开？
log.Infof("new connection from:%v port:%v", host, port)
// session die signal
sess_die := make(chan bool)
//SESSION_DIE 监控有问题.................
// create a write buffer
// 创建写入buffer对象
out := new_buffer(conn, sess_die)
go out.start()
// start one agent for handling packet
//记录goroutine个数，让系统接收到关闭命令后，会阻塞主线程，至少所有agent线程退出，已保证数据落地
wg.Add(1)
go agent(&sess, in, out, sess_die)
//network loop
for {
// solve dead line problem
// 设置读超时时间, 如果在任意一次执行Read syscall 返回的时候，超过这个时间点， 则算超时
conn.SetReadDeadline(time.Now().Add(TCP_READ_DEADLINE * time.Second))
//先读取2个字节头文件长度
n, err := io.ReadFull(conn, header)
if err != nil {
log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
//将2个字节数组转成int16类型, 不丢失精度
size := binary.BigEndian.Uint16(header)
// alloc a byte slice for reading
// 创建一个指定长度的切片，用于存放具体内容
payload := make([]byte, size)
//read msg
n, err = io.ReadFull(conn, payload)
if err != nil {
log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
select {
//接收的数据，转入in通道
case in <- payload: //payload queued
//监听sess_die 通道
case <-sess_die:
log.Warning("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
return
}
}
// 好像没有处理连接超时, 如果玩家连上游戏后，一直未操作，再次链接时，会是新的连接？难道客户端在许久没有操作的情况下，先发一次ping, 如果有响应，继续操作，如果没响应，则执行重连？
// 如果玩家已经退出了游戏，但是通过非正常途径退出的，这时，服务器还保留着该session, 当玩家再次登陆时， 原先的连接何时删除
}
// client protocol handle proxy
func proxy_user_request(sess *Session, p []byte) []byte {
start := time.Now()
defer utils.PrintPanicStack(sess, p)
// 解密
if sess.Flag&SESS_ENCRYPT != 0 {
sess.Decoder.XORKeyStream(p, p)
}
// 封装为reader
reader := packet.Reader(p)
// 读客户端数据包序列号(1,2,3...)
// 客户端发送的数据包必须包含一个自增的序号，必须严格递增
// 加密后，可避免重放攻击-REPLAY-ATTACK
seq_id, err := reader.ReadU32()
if err != nil {
log.Error("read client timestamp failed:", err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 数据包序列号验证
if seq_id != sess.PacketCount {
log.Errorf("illegal packet sequence id:%v should be:%v size:%v", seq_id, sess.PacketCount, len(p)-6)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 读协议号
b, err := reader.ReadS16()
if err != nil {
log.Error("read protocol number failed.")
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 根据协议号断做服务划分
// 协议号的划分采用分割协议区间, 用户可以自定义多个区间，用于转发到不同的后端服务
var ret []byte
if b > MAX_PROTO_NUM {
if err := forward(sess, p[4:]); err != nil {
log.Errorf("service id:%v execute failed, error:%v", b, err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
} else {
if h := client_handler.Handlers[b]; h != nil {
ret = h(sess, reader)
} else {
log.Errorf("service id:%v not bind", b)
sess.Flag |= SESS_KICKED_OUT
return nil
}
}
// 监控协议处理时间
// 监控数值会发送到statsd,格式为:
// API.XXX_REQ = 10ms
elasped := time.Now().Sub(start)
if b != 0 { // 排除心跳包日志
log.Trace("[REQ]", client_handler.RCode[b])
_statter.Timing(1.0, fmt.Sprintf("%v%v", STATSD_PREFIX, client_handler.RCode[b]), elasped)
}
return ret
}
// PIPELINE #1: handleClient
// the goroutine is used for reading incoming PACKETS
// each packet is defined as :
// | 2B size | DATA |
//
func handleClient(conn *net.TCPConn) {
defer utils.PrintPanicStack()
// set socket read buffer
conn.SetReadBuffer(SO_RCVBUF)
// set socket write buffer
conn.SetWriteBuffer(SO_SNDBUF)
// for reading the 2-Byte header
header := make([]byte, 2)
// the input channel for agent()
in := make(chan []byte)
defer func() {
close(in) // session will close
}()
// create a new session object for the connection
// and record it's IP address
var sess Session
host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
log.Error("cannot get remote address:", err)
return
}
sess.IP = net.ParseIP(host)
log.Infof("new connection from:%v port:%v", host, port)
// session die signal, will be triggered by agent()
sess.Die = make(chan struct{})
// create a write buffer
out := new_buffer(conn, sess.Die)
go out.start()
// start agent for PACKET processing
wg.Add(1)
go agent(&sess, in, out)
// read loop
for {
// solve dead link problem:
// physical disconnection without any communcation between client and server
// will cause the read to block FOREVER, so a timeout is a rescue.
conn.SetReadDeadline(time.Now().Add(TCP_READ_DEADLINE * time.Second))
// read 2B header
n, err := io.ReadFull(conn, header)
if err != nil {
log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
size := binary.BigEndian.Uint16(header)
// alloc a byte slice of the size defined in the header for reading data
payload := make([]byte, size)
n, err = io.ReadFull(conn, payload)
if err != nil {
log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
// deliver the data to the input queue of agent()
select {
case in <- payload: // payload queued
case <-sess.Die:
log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
return
}
}
}
func agent(sess *Session, in chan []byte, out *Buffer, sess_die chan bool) {
defer wg.Done()
defer utils.PrintPanicStack()
// init session
//存储玩家异步消息, 让消息超过时, 会阻塞，等待等读取,
//那么如果一直是满的时，玩家好久没上线
//别的玩家发过来的消息会一直发送不过去, 这时会长期阻塞，怎么解决, 发送方设置超时,当许久没发送成功后，弹出对方信箱已满?
// ...... 这里是异步消息，还是离线消息？
sess.MQ = make(chan []byte, DEFAULT_MQ_SIZE)
sess.ConnectTime = time.Now()
sess.LastPacketTime = time.Now()
// minute timer
min_timer := time.After(time.Minute)
// cleanup work
defer func() {
close(sess_die)
// 当session关闭时，连接游戏后段的流需要关闭
if sess.Stream != nil {
sess.Stream.CloseSend()
}
}()
// >> the main message loop <<
for {
select {
case msg, ok := <-in: // packet from network
if !ok {
return
}
//数据包个数++
sess.PacketCount++
//更新接收数据包的时间
sess.PacketTime = time.Now()
//处理用户请求
if result := proxy_user_request(sess, msg); result != nil {
//将结果发送出去
out.send(sess, result)
}
//当处理完之后，将数据包时间设置成上个数据包时间
sess.LastPacketTime = sess.PacketTime
//如果异步通道有消息? (内部发送过来的消息）
case frame := <-sess.MQ:
switch frame.Type {
case spp.Game_Message:
out.send(sess, frame.Message)
//如果是关闭命令, 则设置关闭状态， (用于后台管理)
case spp.Game_Kick:
sess.Flag |= SESS_KICKED_OUT
}
case <-min_timer: //minutes timer
//定时器工作, 用于检测用户行为，防治玩家作弊
timer_work(sess, out)
min_timer = time.After(time.Minute)
case <-die: // server is shuting down...
sess.Flag |= SESS_KICKED_OUT
}
// see if the player should be kicked out.
if sess.Flag&SESS_KICKED_OUT != 0 {
return
}
}
}