// get stored service name
func (p *service_pool) load_names() {
p.service_names = make(map[string]bool)
client := p.client_pool.Get().(*etcd.Client)
defer func() {
p.client_pool.Put(client)
}()
// get the keys under directory
log.Info("reading names:", DEFAULT_NAME_FILE)
resp, err := client.Get(DEFAULT_NAME_FILE, false, false)
if err != nil {
log.Error(err)
return
}
// validation check
if resp.Node.Dir {
log.Error("names is not a file")
return
}
// split names
names := strings.Split(resp.Node.Value, "\n")
log.Info("all service names:", names)
for _, v := range names {
p.service_names[DEFAULT_SERVICE_PATH+"/"+strings.TrimSpace(v)] = true
}
p.enable_name_check = true
}
//---------------------------------------------------------- 读取GridFS文件
func LoadFile(filename string) (ok bool, content []byte) {
ms := _global_ms.Copy()
defer ms.Close()
buf := &bytes.Buffer{}
file, err := ms.DB("").GridFS("fs").Open(filename)
if err != nil {
log.Warning("gridfs", filename, err)
return false, nil
}
n, err := io.Copy(buf, file)
if err != nil {
log.Error("gridfs", filename, n, err)
return false, nil
}
err = file.Close()
if err != nil {
log.Error("gridfs", filename, err)
return false, nil
}
log.Trace("gridfs", filename, "load from GridFS!!")
return true, buf.Bytes()
}
// connect to all services
func (p *service_pool) connect_all(directory string) {
client := p.client_pool.Get().(*etcd.Client)
defer func() {
p.client_pool.Put(client)
}()
// get the keys under directory
log.Info("connecting services under:", directory)
resp, err := client.Get(directory, true, true)
if err != nil {
log.Error(err)
return
}
// validation check
if !resp.Node.Dir {
log.Error("not a directory")
return
}
for _, node := range resp.Node.Nodes {
if node.Dir { // service directory
for _, service := range node.Nodes {
p.add_service(service.Key, service.Value)
}
} else {
log.Warning("malformed service directory:", node.Key)
}
}
log.Info("services add complete")
}
// generate an unique uuid
func (s *server) GetUUID(context.Context, *pb.Snowflake_NullRequest) (*pb.Snowflake_UUID, error) {
s.Lock()
defer s.Unlock()
// get a correct serial number
t := s.ts()
if t < s.last_ts { // clock shift backward
log.Error("clock shift happened, waiting until the clock moving to the next millisecond.")
t = s.wait_ms(s.last_ts)
}
if s.last_ts == t { // same millisecond
s.sn = (s.sn + 1) & SN_MASK
if s.sn == 0 { // serial number overflows, wait until next ms
t = s.wait_ms(s.last_ts)
}
} else { // new millsecond, reset serial number to 0
s.sn = 0
}
// remember last timestamp
s.last_ts = t
// generate uuid, format:
//
// 0 0.................0 0..............0 0........0
// 1-bit 41bit timestamp 10bit machine-id 12bit sn
var uuid uint64
uuid |= (uint64(t) & TS_MASK) << 22
uuid |= s.machine_id
uuid |= s.sn
return &pb.Snowflake_UUID{uuid}, nil
}
func (ps *PubSub) init() {
ps.ch_msg = make(chan interface{})
call := func(rf reflect.Value, in []reflect.Value) {
defer func() { // do not let callback panic the pubsub
if err := recover(); err != nil {
log.Error(err)
}
}()
rf.Call(in)
}
go func() {
for v := range ps.ch_msg {
rv := reflect.ValueOf(v)
ps.Lock()
for _, sub := range ps.subs {
rf := reflect.ValueOf(sub.f)
if rv.Type() == reflect.ValueOf(sub.f).Type().In(0) { // parameter type match
call(rf, []reflect.Value{rv})
}
}
ps.Unlock()
}
}()
}
// uuid generator
func (s *server) uuid_task() {
var sn uint64 // 12-bit serial no
var last_ts int64 // last timestamp
for {
ret := <-s.ch_proc
// get a correct serial number
t := ts()
if t < last_ts { // clock shift backward
log.Error("clock shift happened, waiting until the clock moving to the next millisecond.")
t = s.wait_ms(last_ts)
}
if last_ts == t { // same millisecond
sn = (sn + 1) & SN_MASK
if sn == 0 { // serial number overflows, wait until next ms
t = s.wait_ms(last_ts)
}
} else { // new millsecond, reset serial number to 0
sn = 0
}
// remember last timestamp
last_ts = t
// generate uuid, format:
//
// 0 0.................0 0..............0 0........0
// 1-bit 41bit timestamp 10bit machine-id 12bit sn
var uuid uint64
uuid |= (uint64(t) & TS_MASK) << 22
uuid |= s.machine_id
uuid |= sn
ret <- uuid
}
}
// get next value of a key, like auto-increment in mysql
func (s *server) Next(ctx context.Context, in *pb.Snowflake_Key) (*pb.Snowflake_Value, error) {
client := s.client_pool.Get().(*etcd.Client)
defer func() {
s.client_pool.Put(client)
}()
key := PATH + in.Name
for i := 0; i < RETRY_MAX; i++ {
// get the key
resp, err := client.Get(key, false, false)
if err != nil {
log.Critical(err)
return nil, errors.New("Key not exists, need to create first")
}
// get prevValue & prevIndex
prevValue, err := strconv.Atoi(resp.Node.Value)
if err != nil {
log.Critical(err)
return nil, errors.New("marlformed value")
}
prevIndex := resp.Node.ModifiedIndex
// CAS
resp, err = client.CompareAndSwap(key, fmt.Sprint(prevValue+1), 0, resp.Node.Value, prevIndex)
if err != nil {
log.Error(err)
<-time.After(RETRY_DELAY)
continue
}
return &pb.Snowflake_Value{int64(prevValue + 1)}, nil
}
return nil, errors.New("etcd server busy")
}
func (arch *Archiver) archive_task() {
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGTERM)
timer := time.After(REDO_ROTATE_INTERVAL)
db := arch.new_redolog()
for {
select {
case msg := <-arch.pending:
var record map[string]interface{}
err := msgpack.Unmarshal(msg, &record)
if err != nil {
log.Error(err)
continue
}
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(BOLTDB_BUCKET))
err := b.Put([]byte(fmt.Sprint(record["TS"])), msg)
return err
})
case <-timer:
db.Close()
// rotate redolog
db = arch.new_redolog()
timer = time.After(REDO_ROTATE_INTERVAL)
case <-sig:
db.Close()
log.Info("SIGTERM")
os.Exit(0)
}
}
}
// unlock a key on etcd, returns false if the key cannot be successfully unlocked
func (m *Mutex) Unlock() bool {
client := _client_pool.Get().(*etcd.Client)
_, err := client.CompareAndDelete(LOCK_PATH+m.key, VALUE, m.prevIndex)
if err != nil {
log.Error(SERVICE, err)
return false
}
return true
}
func (s *server) query(ip net.IP) *City {
city := &City{}
err := s.mmdb.Lookup(ip, city)
if err != nil {
log.Error(err)
return nil
}
return city
}
// 玩家登陆过程
func P_user_login_req(sess *Session, reader *packet.Packet) []byte {
// TODO: 登陆鉴权
sess.UserId = 1
// TODO: 选择登陆服务器
sess.GSID = DEFAULT_GSID
// 选服
cli, err := sp.GetServiceWithId(sp.SERVICE_GAME, sess.GSID)
if err != nil {
log.Critical(err)
return nil
}
// type assertion
game_cli, ok := cli.(spp.GameServiceClient)
if !ok {
log.Critical("cannot do type assertion on: %v", sess.GSID)
return nil
}
// 开启到游戏服的流
// TODO: 处理context，设定超时
stream, err := game_cli.Stream(context.Background())
if err != nil {
log.Critical(err)
return nil
}
sess.Stream = stream
// 在game注册
// TODO: 新用户的创建由game处理
sess.Stream.Send(&spp.Game_Frame{Type: spp.Game_Register, UserId: sess.UserId})
// 读取GAME返回消息
fetcher_task := func(sess *Session) {
for {
in, err := sess.Stream.Recv()
if err == io.EOF { // 流关闭
log.Trace(err)
return
}
if err != nil {
log.Error(err)
return
}
sess.MQ <- *in
}
}
go fetcher_task(sess)
return packet.Pack(Code["user_login_ack"], user_snapshot{F_uid: sess.UserId}, nil)
}
// 玩家1分钟定时器
func timer_work(sess *Session, out *Buffer) {
// 发包频率控制，太高的RPS直接踢掉
interval := time.Now().Sub(sess.ConnectTime).Minutes()
if interval >= 1 { // 登录时长超过1分钟才开始统计rpm。防脉冲
rpm := float64(sess.PacketCount) / interval
if rpm > RPM_LIMIT {
sess.Flag |= SESS_KICKED_OUT
log.Error("玩家RPM太高 RPM:", rpm)
return
}
}
}
// 玩家登陆过程
func P_user_login_req(sess *Session, reader *packet.Packet) []byte {
// TODO: 登陆鉴权
sess.UserId = 1
// TODO: 选择登陆服务器
sess.GSID = DEFAULT_GSID
// 选服
conn := sp.GetServiceWithId(sp.DEFAULT_SERVICE_PATH+"/game", sess.GSID)
if conn == nil {
log.Critical("cannot get game service:", sess.GSID)
return nil
}
cli := pb.NewGameServiceClient(conn)
// 开启到游戏服的流
// TODO: 处理context，设定超时
stream, err := cli.Stream(context.Background())
if err != nil {
log.Critical(err)
return nil
}
sess.Stream = stream
// 在game注册
// TODO: 新用户的创建由game处理
sess.Stream.Send(&pb.Game_Frame{Type: pb.Game_Register, UserId: sess.UserId})
// 读取GAME返回消息
fetcher_task := func(sess *Session) {
for {
in, err := sess.Stream.Recv()
if err == io.EOF { // 流关闭
log.Trace(err)
return
}
if err != nil {
log.Error(err)
return
}
select {
case sess.MQ <- *in:
case <-sess.Die:
}
}
}
go fetcher_task(sess)
return packet.Pack(Code["user_login_succeed_ack"], S_user_snapshot{F_uid: sess.UserId}, nil)
}
func PrintPanicStack(extras ...interface{}) {
if x := recover(); x != nil {
log.Error(x)
i := 0
funcName, file, line, ok := runtime.Caller(i)
for ok {
log.Errorf("frame %v:[func:%v,file:%v,line:%v]\n", i, runtime.FuncForPC(funcName).Name(), file, line)
i++
funcName, file, line, ok = runtime.Caller(i)
}
for k := range extras {
log.Errorf("EXRAS#%v DATA:%v\n", k, spew.Sdump(extras[k]))
}
}
}
func init() {
addr := DEFAULT_STATSD_HOST
if env := os.Getenv(ENV_STATSD); env != "" {
addr = env
}
s, err := g2s.Dial("udp", addr)
if err == nil {
_statter = s
} else {
_statter = g2s.Noop()
log.Error(err)
}
go pprof_task()
}
// 玩家登陆过程
func P_user_login_req(sess *Session, reader *packet.Packet) []byte {
// TODO: 登陆鉴权
// 简单鉴权可以在agent直接完成，通常公司都存在一个用户中心服务器用于鉴权
sess.UserId = 1
// TODO: 选择GAME服务器
// 选服策略依据业务进行，比如小服可以固定选取某台，大服可以采用HASH或一致性HASH
sess.GSID = DEFAULT_GSID
// 连接到已选定GAME服务器
conn := sp.GetServiceWithId(sp.DEFAULT_SERVICE_PATH+"/game", sess.GSID)
if conn == nil {
log.Critical("cannot get game service:", sess.GSID)
return nil
}
cli := pb.NewGameServiceClient(conn)
// 开启到游戏服的流
ctx := metadata.NewContext(context.Background(), metadata.New(map[string]string{"userid": fmt.Sprint(sess.UserId)}))
stream, err := cli.Stream(ctx)
if err != nil {
log.Critical(err)
return nil
}
sess.Stream = stream
// 读取GAME返回消息的goroutine
fetcher_task := func(sess *Session) {
for {
in, err := sess.Stream.Recv()
if err == io.EOF { // 流关闭
log.Trace(err)
return
}
if err != nil {
log.Error(err)
return
}
select {
case sess.MQ <- *in:
case <-sess.Die:
}
}
}
go fetcher_task(sess)
return packet.Pack(Code["user_login_succeed_ack"], S_user_snapshot{F_uid: sess.UserId}, nil)
}
// forward messages to game server
func forward(sess *Session, p []byte) error {
frame := &pb.Game_Frame{
Type: pb.Game_Message,
Message: p,
}
// check stream
if sess.Stream == nil {
return ERROR_STREAM_NOT_OPEN
}
// forward the frame to game
if err := sess.Stream.Send(frame); err != nil {
log.Error(err)
return err
}
return nil
}
// subscribe to an endpoint & receive server streams
func (s *server) Subscribe(p *Chat_Id, stream ChatService_SubscribeServer) error {
// read endpoint
ep := s.read_ep(p.Id)
if ep == nil {
log.Errorf("cannot find endpoint %v", p)
return ERROR_NOT_EXISTS
}
// send history chat messages
msgs := ep.Read()
for k := range msgs {
if err := stream.Send(&msgs[k]); err != nil {
return nil
}
}
// create subscriber
e := make(chan error, 1)
var once sync.Once
f := NewSubscriber(func(msg *Chat_Message) {
if err := stream.Send(msg); err != nil {
once.Do(func() { // protect for channel blocking
e <- err
})
}
})
// subscribe to the endpoint
log.Tracef("subscribe to:%v", p.Id)
ep.ps.Sub(f)
defer func() {
ep.ps.Leave(f)
log.Tracef("leave from:%v", p.Id)
}()
// client send cancel to stop receiving, see service_test.go for example
select {
case <-stream.Context().Done():
case <-e:
log.Error(e)
}
return nil
}
// lock a key on etcd and return mutex,returns nil if cannot lock the key
func Lock(key string) *Mutex {
m := &Mutex{}
m.key = key
client := _client_pool.Get().(*etcd.Client)
defer func() {
_client_pool.Put(client)
}()
for i := 0; i < RETRY_MAX; i++ {
// create a key with ttl
resp, err := client.Create(LOCK_PATH+key, VALUE, TTL)
if err != nil {
log.Error(SERVICE, err)
<-time.After(RETRY_WAIT)
continue
}
// remember index
m.prevIndex = resp.Node.ModifiedIndex
return m
}
return nil
}
func (s *server) init_machine_id() {
client := s.client_pool.Get().(*etcd.Client)
defer func() {
s.client_pool.Put(client)
}()
for i := 0; i < RETRY_MAX; i++ {
// get the key
resp, err := client.Get(UUID_KEY, false, false)
if err != nil {
log.Critical(err)
os.Exit(-1)
}
// get prevValue & prevIndex
prevValue, err := strconv.Atoi(resp.Node.Value)
if err != nil {
log.Critical(err)
os.Exit(-1)
}
prevIndex := resp.Node.ModifiedIndex
// CAS
resp, err = client.CompareAndSwap(UUID_KEY, fmt.Sprint(prevValue+1), 0, resp.Node.Value, prevIndex)
if err != nil {
log.Error(err)
<-time.After(RETRY_DELAY)
continue
}
// record serial number of this service, already shifted
s.machine_id = (uint64(prevValue+1) & MACHINE_ID_MASK) << 12
return
}
// failed to get machine id, exit
os.Exit(-1)
}
func (ns *numbers) init(path string) {
ns.tables = make(map[string]*table)
kapi := etcdclient.KeysAPI()
resp, err := kapi.Get(context.Background(), path, nil)
if err != nil {
log.Error(err)
return
}
// 解码xlsx
xlsx_bin, err := base64.StdEncoding.DecodeString(resp.Node.Value)
if err != nil {
log.Critical(err)
return
}
// 读取xlsx
xlsx_reader, err := xlsx.OpenBinary(xlsx_bin)
if err != nil {
log.Critical(err)
return
}
ns.parse(xlsx_reader.Sheets)
}
// PIPELINE #1: handleClient
// the goroutine is used for reading incoming PACKETS
// each packet is defined as :
// | 2B size | DATA |
//
func handleClient(conn *net.TCPConn) {
defer utils.PrintPanicStack()
// set socket read buffer
conn.SetReadBuffer(SO_RCVBUF)
// set socket write buffer
conn.SetWriteBuffer(SO_SNDBUF)
// for reading the 2-Byte header
header := make([]byte, 2)
// the input channel for agent()
in := make(chan []byte)
defer func() {
close(in) // session will close
}()
// create a new session object for the connection
// and record it's IP address
var sess Session
host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
log.Error("cannot get remote address:", err)
return
}
sess.IP = net.ParseIP(host)
log.Infof("new connection from:%v port:%v", host, port)
// session die signal, will be triggered by agent()
sess.Die = make(chan struct{})
// create a write buffer
out := new_buffer(conn, sess.Die)
go out.start()
// start agent for PACKET processing
wg.Add(1)
go agent(&sess, in, out)
// read loop
for {
// solve dead link problem:
// physical disconnection without any communcation between client and server
// will cause the read to block FOREVER, so a timeout is a rescue.
conn.SetReadDeadline(time.Now().Add(TCP_READ_DEADLINE * time.Second))
// read 2B header
n, err := io.ReadFull(conn, header)
if err != nil {
log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
size := binary.BigEndian.Uint16(header)
// alloc a byte slice of the size defined in the header for reading data
payload := make([]byte, size)
n, err = io.ReadFull(conn, payload)
if err != nil {
log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
// deliver the data to the input queue of agent()
select {
case in <- payload: // payload queued
case <-sess.Die:
log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
return
}
}
}
// start a goroutine when a new connection is accepted
func handleClient(conn *net.TCPConn) {
defer utils.PrintPanicStack()
// set per-connection socket buffer
conn.SetReadBuffer(SO_RCVBUF)
// set initial socket buffer
conn.SetWriteBuffer(SO_SNDBUF)
// initial network control struct
header := make([]byte, 2)
in := make(chan []byte)
defer func() {
close(in) // session will close
}()
// create a new session object for the connection
var sess Session
host, port, err := net.SplitHostPort(conn.RemoteAddr().String())
if err != nil {
log.Error("cannot get remote address:", err)
return
}
sess.IP = net.ParseIP(host)
log.Infof("new connection from:%v port:%v", host, port)
// session die signal
sess.Die = make(chan struct{})
// create a write buffer
out := new_buffer(conn, sess.Die)
go out.start()
// start one agent for handling packet
wg.Add(1)
go agent(&sess, in, out)
// network loop
for {
// solve dead link problem
conn.SetReadDeadline(time.Now().Add(TCP_READ_DEADLINE * time.Second))
n, err := io.ReadFull(conn, header)
if err != nil {
log.Warningf("read header failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
size := binary.BigEndian.Uint16(header)
// alloc a byte slice for reading
payload := make([]byte, size)
// read msg
n, err = io.ReadFull(conn, payload)
if err != nil {
log.Warningf("read payload failed, ip:%v reason:%v size:%v", sess.IP, err, n)
return
}
select {
case in <- payload: // payload queued
case <-sess.Die:
log.Warningf("connection closed by logic, flag:%v ip:%v", sess.Flag, sess.IP)
return
}
}
}
// client protocol handle proxy
func proxy_user_request(sess *Session, p []byte) []byte {
start := time.Now()
defer utils.PrintPanicStack(sess, p)
// 解密
if sess.Flag&SESS_ENCRYPT != 0 {
sess.Decoder.XORKeyStream(p, p)
}
// 封装为reader
reader := packet.Reader(p)
// 读客户端数据包序列号(1,2,3...)
// 可避免重放攻击-REPLAY-ATTACK
seq_id, err := reader.ReadU32()
if err != nil {
log.Error("read client timestamp failed:", err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 数据包序列号验证
if seq_id != sess.PacketCount {
log.Errorf("illegal packet sequence id:%v should be:%v size:%v", seq_id, sess.PacketCount, len(p)-6)
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 读协议号
b, err := reader.ReadS16()
if err != nil {
log.Error("read protocol number failed.")
sess.Flag |= SESS_KICKED_OUT
return nil
}
// 根据协议号断做服务划分
var ret []byte
if b > MAX_PROTO_NUM {
if err := forward(sess, p[4:]); err != nil {
log.Errorf("service id:%v execute failed, error:%v", b, err)
sess.Flag |= SESS_KICKED_OUT
return nil
}
} else {
if h := client_handler.Handlers[b]; h != nil {
ret = h(sess, reader)
} else {
log.Errorf("service id:%v not bind", b)
sess.Flag |= SESS_KICKED_OUT
return nil
}
}
// 统计处理时间
elasped := time.Now().Sub(start)
if b != 0 { // 排除心跳包日志
log.Trace("[REQ]", client_handler.RCode[b])
_statter.Timing(1.0, fmt.Sprintf("%v%v", STATSD_PREFIX, client_handler.RCode[b]), elasped)
}
return ret
}
func checkErr(err error) {
if err != nil {
log.Error(err)
panic("error occured in protocol module")
}
}