// Accept accepts connections on the listener and serves requests
// for each incoming connection. Accept blocks; the caller typically
// invokes it in a go statement.
func acceptTCP(server *Server, lis *net.TCPListener) {
var (
conn *net.TCPConn
err error
r int
)
for {
if conn, err = lis.AcceptTCP(); err != nil {
// if listener close then return
log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
return
}
if err = conn.SetKeepAlive(server.Options.TCPKeepalive); err != nil {
log.Error("conn.SetKeepAlive() error(%v)", err)
return
}
if err = conn.SetReadBuffer(server.Options.TCPRcvbuf); err != nil {
log.Error("conn.SetReadBuffer() error(%v)", err)
return
}
if err = conn.SetWriteBuffer(server.Options.TCPSndbuf); err != nil {
log.Error("conn.SetWriteBuffer() error(%v)", err)
return
}
go serveTCP(server, conn, r)
if r++; r == maxInt {
r = 0
}
}
}
func PushAll(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method Not Allowed", 405)
return
}
var (
bodyBytes []byte
body string
err error
res = map[string]interface{}{"ret": OK}
)
defer retPWrite(w, r, res, &body, time.Now())
if bodyBytes, err = ioutil.ReadAll(r.Body); err != nil {
log.Error("ioutil.ReadAll() failed (%v)", err)
res["ret"] = InternalErr
return
}
body = string(bodyBytes)
// push all
if err := broadcastKafka(bodyBytes); err != nil {
log.Error("broadcastKafka(\"%s\") error(%s)", body, err)
res["ret"] = InternalErr
return
}
res["ret"] = OK
return
}
// InitTCP listen all tcp.bind and start accept connections.
func InitTCP(addrs []string, accept int) (err error) {
var (
bind string
listener *net.TCPListener
addr *net.TCPAddr
)
for _, bind = range addrs {
if addr, err = net.ResolveTCPAddr("tcp4", bind); err != nil {
log.Error("net.ResolveTCPAddr(\"tcp4\", \"%s\") error(%v)", bind, err)
return
}
if listener, err = net.ListenTCP("tcp4", addr); err != nil {
log.Error("net.ListenTCP(\"tcp4\", \"%s\") error(%v)", bind, err)
return
}
if Debug {
log.Debug("start tcp listen: \"%s\"", bind)
}
// split N core accept
for i := 0; i < accept; i++ {
go acceptTCP(DefaultServer, listener)
}
}
return
}
// retPWrite marshal the result and write to client(post).
func retPWrite(w http.ResponseWriter, r *http.Request, res map[string]interface{}, body *string, start time.Time) {
data, err := json.Marshal(res)
if err != nil {
log.Error("json.Marshal(\"%v\") error(%v)", res, err)
return
}
dataStr := string(data)
if _, err := w.Write([]byte(dataStr)); err != nil {
log.Error("w.Write(\"%s\") error(%v)", dataStr, err)
}
log.Info("req: \"%s\", post: \"%s\", res:\"%s\", ip:\"%s\", time:\"%fs\"", r.URL.String(), *body, dataStr, r.RemoteAddr, time.Now().Sub(start).Seconds())
}
func httpListen(mux *http.ServeMux, network, addr string) {
httpServer := &http.Server{Handler: mux, ReadTimeout: Conf.HTTPReadTimeout, WriteTimeout: Conf.HTTPWriteTimeout}
httpServer.SetKeepAlivesEnabled(true)
l, err := net.Listen(network, addr)
if err != nil {
log.Error("net.Listen(\"%s\", \"%s\") error(%v)", network, addr, err)
panic(err)
}
if err := httpServer.Serve(l); err != nil {
log.Error("server.Serve() error(%v)", err)
panic(err)
}
}
func rpcListen(network, addr string) {
l, err := net.Listen(network, addr)
if err != nil {
log.Error("net.Listen(\"%s\", \"%s\") error(%v)", network, addr, err)
panic(err)
}
// if process exit, then close the rpc addr
defer func() {
log.Info("listen rpc: \"%s\" close", addr)
if err := l.Close(); err != nil {
log.Error("listener.Close() error(%v)", err)
}
}()
rpc.Accept(l)
}
// Dial connects to an RPC server at the specified network address.
func (c *Client) dial() (err error) {
var conn net.Conn
conn, err = net.DialTimeout(c.options.Proto, c.options.Addr, dialTimeout)
if err != nil {
log.Error("net.Dial(%s, %s), error(%v)", c.options.Proto, c.options.Addr, err)
} else {
c.Client = rpc.NewClient(conn)
}
return
}
func handleError() {
var (
err *sarama.ProducerError
)
for {
err = <-producer.Errors()
if err != nil {
log.Error("producer message error, partition:%d offset:%d key:%v valus:%s error(%v)", err.Msg.Partition, err.Msg.Offset, err.Msg.Key, err.Msg.Value, err.Err)
}
}
}
func Push(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method Not Allowed", 405)
return
}
var (
body string
serverId int32
keys []string
subKeys map[int32][]string
bodyBytes []byte
userId int64
err error
uidStr = r.URL.Query().Get("uid")
res = map[string]interface{}{"ret": OK}
)
defer retPWrite(w, r, res, &body, time.Now())
if bodyBytes, err = ioutil.ReadAll(r.Body); err != nil {
log.Error("ioutil.ReadAll() failed (%s)", err)
res["ret"] = InternalErr
return
}
body = string(bodyBytes)
if userId, err = strconv.ParseInt(uidStr, 10, 64); err != nil {
log.Error("strconv.Atoi(\"%s\") error(%v)", uidStr, err)
res["ret"] = InternalErr
return
}
subKeys = genSubKey(userId)
for serverId, keys = range subKeys {
if err = mpushKafka(serverId, keys, bodyBytes); err != nil {
res["ret"] = InternalErr
return
}
}
res["ret"] = OK
return
}
// {"m":{"test":1},"u":"1,2,3"}
func Pushs(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method Not Allowed", 405)
return
}
var (
body string
bodyBytes []byte
serverId int32
userIds []int64
err error
res = map[string]interface{}{"ret": OK}
subKeys map[int32][]string
keys []string
)
defer retPWrite(w, r, res, &body, time.Now())
if bodyBytes, err = ioutil.ReadAll(r.Body); err != nil {
log.Error("ioutil.ReadAll() failed (%s)", err)
res["ret"] = InternalErr
return
}
body = string(bodyBytes)
if bodyBytes, userIds, err = parsePushsBody(bodyBytes); err != nil {
log.Error("parsePushsBody(\"%s\") error(%s)", body, err)
res["ret"] = InternalErr
return
}
subKeys = genSubKeys(userIds)
for serverId, keys = range subKeys {
if err = mpushKafka(serverId, keys, bodyBytes); err != nil {
res["ret"] = InternalErr
return
}
}
res["ret"] = OK
return
}
func PushRoom(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method Not Allowed", 405)
return
}
var (
bodyBytes []byte
body string
rid int
err error
param = r.URL.Query()
res = map[string]interface{}{"ret": OK}
)
defer retPWrite(w, r, res, &body, time.Now())
if bodyBytes, err = ioutil.ReadAll(r.Body); err != nil {
log.Error("ioutil.ReadAll() failed (%v)", err)
res["ret"] = InternalErr
return
}
body = string(bodyBytes)
ridStr := param.Get("rid")
enable, _ := strconv.ParseBool(param.Get("ensure"))
// push room
if rid, err = strconv.Atoi(ridStr); err != nil {
log.Error("strconv.Atoi(\"%s\") error(%v)", ridStr, err)
res["ret"] = InternalErr
return
}
if err = broadcastRoomKafka(int32(rid), bodyBytes, enable); err != nil {
log.Error("broadcastRoomKafka(\"%s\",\"%s\",\"%d\") error(%s)", rid, body, enable, err)
res["ret"] = InternalErr
return
}
res["ret"] = OK
return
}
func InitRPCPush(addrs []string) (err error) {
var (
bind string
network, addr string
c = &PushRPC{}
)
rpc.Register(c)
for _, bind = range addrs {
if network, addr, err = inet.ParseNetwork(bind); err != nil {
log.Error("inet.ParseNetwork() error(%v)", err)
return
}
go rpcListen(network, addr)
}
return
}
func InitHTTP() (err error) {
// http listen
var network, addr string
for i := 0; i < len(Conf.HTTPAddrs); i++ {
httpServeMux := http.NewServeMux()
httpServeMux.HandleFunc("/1/push", Push)
httpServeMux.HandleFunc("/1/pushs", Pushs)
httpServeMux.HandleFunc("/1/push/all", PushAll)
httpServeMux.HandleFunc("/1/push/room", PushRoom)
httpServeMux.HandleFunc("/1/server/del", DelServer)
httpServeMux.HandleFunc("/1/count", Count)
log.Info("start http listen:\"%s\"", Conf.HTTPAddrs[i])
if network, addr, err = inet.ParseNetwork(Conf.HTTPAddrs[i]); err != nil {
log.Error("inet.ParseNetwork() error(%v)", err)
return
}
go httpListen(httpServeMux, network, addr)
}
return
}
func DelServer(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Method Not Allowed", 405)
return
}
var (
err error
serverStr = r.URL.Query().Get("server")
server int64
res = map[string]interface{}{"ret": OK}
)
if server, err = strconv.ParseInt(serverStr, 10, 32); err != nil {
log.Error("strconv.Atoi(\"%s\") error(%v)", serverStr, err)
res["ret"] = InternalErr
return
}
defer retWrite(w, r, res, time.Now())
if err = delServer(int32(server)); err != nil {
res["ret"] = InternalErr
return
}
return
}
// ping ping the rpc connect and reconnect when has an error.
func (c *Client) Ping(serviceMethod string) {
var (
arg = proto.NoArg{}
reply = proto.NoReply{}
err error
)
for {
select {
case <-c.quit:
goto closed
return
default:
}
if c.Client != nil && c.err == nil {
// ping
if err = c.Call(serviceMethod, &arg, &reply); err != nil {
c.err = err
if err != rpc.ErrShutdown {
c.Client.Close()
}
log.Error("client.Call(%s, arg, reply) error(%v)", serviceMethod, err)
}
} else {
// reconnect
if err = c.dial(); err == nil {
// reconnect ok
c.err = nil
log.Info("client reconnect %s ok", c.options.Addr)
}
}
time.Sleep(pingDuration)
}
closed:
if c.Client != nil {
c.Client.Close()
}
}
func startClient(key string) {
//time.Sleep(time.Duration(mrand.Intn(30)) * time.Second)
quit := make(chan bool, 1)
defer close(quit)
conn, err := net.Dial("tcp", os.Args[3])
if err != nil {
log.Error("net.Dial(\"%s\") error(%v)", os.Args[3], err)
return
}
seqId := int32(0)
wr := bufio.NewWriter(conn)
rd := bufio.NewReader(conn)
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
proto.SeqId = seqId
proto.Body = []byte(key)
if err = tcpWriteProto(wr, proto); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
if err = tcpReadProto(rd, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
log.Debug("key:%s auth ok, proto: %v", key, proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = tcpWriteProto(wr, proto1); err != nil {
log.Error("key:%s tcpWriteProto() error(%v)", key, err)
return
}
log.Debug("key:%s Write heartbeat", key)
// test heartbeat
time.Sleep(heart)
seqId++
select {
case <-quit:
return
default:
}
}
}()
// reader
for {
if err = tcpReadProto(rd, proto); err != nil {
log.Error("key:%s tcpReadProto() error(%v)", key, err)
quit <- true
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("key:%s receive heartbeat", key)
if err = conn.SetReadDeadline(time.Now().Add(heart + 60*time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
quit <- true
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Info("key:%s msg: %s", key, string(proto.Body))
atomic.AddInt64(&countDown, 1)
}
}
}
// dispatch accepts connections on the listener and serves requests
// for each incoming connection. dispatch blocks; the caller typically
// invokes it in a go statement.
func (server *Server) dispatchTCP(key string, conn *net.TCPConn, wr *bufio.Writer, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
var (
p *proto.Proto
err error
)
if Debug {
log.Debug("key: %s start dispatch tcp goroutine", key)
}
for {
p = ch.Ready()
if Debug {
log.Debug("key:%s dispatch msg:%v", key, *p)
}
switch p {
case proto.ProtoFinish:
if Debug {
log.Debug("key: %s wakeup exit dispatch goroutine", key)
}
goto failed
case proto.ProtoReady:
// fetch message from svrbox(client send)
for {
if p, err = ch.CliProto.Get(); err != nil {
err = nil // must be empty error
break
}
//LogSlow(SlowLogTypeReceive, key, p)
if err = p.WriteTCP(wr); err != nil {
goto failed
}
p.Body = nil // avoid memory leak
ch.CliProto.GetAdv()
}
default:
// server send
//LogSlow(SlowLogTypeReceive, key, p)
if err = p.WriteTCP(wr); err != nil {
goto failed
}
}
// only hungry flush response
if err = wr.Flush(); err != nil {
break
}
}
failed:
if err != nil {
log.Error("key: %s dispatch tcp error(%v)", key, err)
}
conn.Close()
wp.Put(wb)
// must ensure all channel message discard, for reader won't blocking Signal
for {
if p == proto.ProtoFinish {
break
}
p = ch.Ready()
}
if Debug {
log.Debug("key: %s dispatch goroutine exit", key)
}
return
}
func initWebsocket() {
origin := "http://" + Conf.WebsocketAddr + "/sub"
url := "ws://" + Conf.WebsocketAddr + "/sub"
conn, err := websocket.Dial(url, "", origin)
if err != nil {
log.Error("websocket.Dial(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
seqId := int32(0)
proto.SeqId = seqId
proto.Body = []byte("{\"test\":1}")
if err = websocketWriteProto(conn, proto); err != nil {
log.Error("websocketWriteProto() error(%v)", err)
return
}
if err = websocketReadProto(conn, proto); err != nil {
log.Error("websocketReadProto() error(%v)", err)
return
}
log.Debug("auth ok, proto: %v", proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
// test heartbeat
//time.Sleep(time.Second * 31)
seqId++
// op_test
proto1.Operation = OP_TEST
proto1.SeqId = seqId
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
seqId++
time.Sleep(10000 * time.Millisecond)
}
}()
// reader
for {
if err = websocketReadProto(conn, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("receive heartbeat")
if err = conn.SetReadDeadline(time.Now().Add(25 * time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Debug("body: %s", string(proto.Body))
}
}
}
// TODO linger close?
func (server *Server) serveTCP(conn *net.TCPConn, rp, wp *bytes.Pool, tr *itime.Timer) {
var (
err error
key string
hb time.Duration // heartbeat
p *proto.Proto
b *Bucket
trd *itime.TimerData
rb = rp.Get()
wb = wp.Get()
ch = NewChannel(server.Options.CliProto, server.Options.SvrProto, define.NoRoom)
rr = &ch.Reader
wr = &ch.Writer
)
ch.Reader.ResetBuffer(conn, rb.Bytes())
ch.Writer.ResetBuffer(conn, wb.Bytes())
// handshake
trd = tr.Add(server.Options.HandshakeTimeout, func() {
conn.Close()
})
// must not setadv, only used in auth
if p, err = ch.CliProto.Set(); err == nil {
if key, ch.RoomId, hb, err = server.authTCP(rr, wr, p); err == nil {
b = server.Bucket(key)
err = b.Put(key, ch, tr)
}
}
if err != nil {
conn.Close()
rp.Put(rb)
wp.Put(wb)
tr.Del(trd)
log.Error("key: %s handshake failed error(%v)", key, err)
return
}
trd.Key = key
tr.Set(trd, hb)
// hanshake ok start dispatch goroutine
go server.dispatchTCP(key, conn, wr, wp, wb, ch)
for {
if p, err = ch.CliProto.Set(); err != nil {
break
}
if err = p.ReadTCP(rr); err != nil {
break
}
//p.Time = *globalNowTime
if p.Operation == define.OP_HEARTBEAT {
tr.Set(trd, hb)
p.Body = nil
p.Operation = define.OP_HEARTBEAT_REPLY
if Debug {
log.Debug("key: %s receive heartbeat", key)
}
} else {
if err = server.operator.Operate(p); err != nil {
break
}
}
ch.CliProto.SetAdv()
ch.Signal()
}
if err != nil && err != io.EOF {
log.Error("key: %s server tcp failed error(%v)", key, err)
}
b.Del(key)
tr.Del(trd)
rp.Put(rb)
conn.Close()
ch.Close()
if err = server.operator.Disconnect(key, ch.RoomId); err != nil {
log.Error("key: %s operator do disconnect error(%v)", key, err)
}
if Debug {
log.Debug("key: %s server tcp goroutine exit", key)
}
return
}
func initWebsocketTLS() {
origin := "https://" + Conf.WebsocketAddr + "/sub"
url := "wss://" + Conf.WebsocketAddr + "/sub"
conf, err := websocket.NewConfig(url, origin)
if err != nil {
log.Error("websocket.NewConfig(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
roots := x509.NewCertPool()
certPem, err := ioutil.ReadFile(Conf.CertFile)
if err != nil {
panic(err)
}
ok := roots.AppendCertsFromPEM(certPem)
if !ok {
panic("failed to parse root certificate")
}
tlsConf := &tls.Config{
//InsecureSkipVerify: true,
RootCAs: roots,
ServerName: "bili.com",
}
conf.TlsConfig = tlsConf
conn, err := websocket.DialConfig(conf)
if err != nil {
log.Error("websocket.Dial(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
seqId := int32(0)
proto.SeqId = seqId
proto.Body = []byte("{\"test\":1}")
if err = websocketWriteProto(conn, proto); err != nil {
log.Error("websocketWriteProto() error(%v)", err)
return
}
if err = websocketReadProto(conn, proto); err != nil {
log.Error("websocketReadProto() error(%v)", err)
return
}
log.Debug("auth ok, proto: %v", proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
// test heartbeat
//time.Sleep(time.Second * 31)
seqId++
// op_test
proto1.Operation = OP_TEST
proto1.SeqId = seqId
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
seqId++
time.Sleep(10000 * time.Millisecond)
}
}()
// reader
for {
if err = websocketReadProto(conn, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("receive heartbeat")
if err = conn.SetReadDeadline(time.Now().Add(25 * time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Debug("body: %s", string(proto.Body))
}
}
}