func (t *Timer) del(td *TimerData) {
var (
i = td.index
last = len(t.timers) - 1
)
if i < 0 || i > last || t.timers[i] != td {
// already remove, usually by expire
if Debug {
log.Debug("timer del i: %d, last: %d, %p", i, last, td)
}
return
}
if i != last {
t.swap(i, last)
t.down(i, last)
t.up(i)
}
// remove item is the last node
t.timers[last].index = -1 // for safety
t.timers = t.timers[:last]
if Debug {
log.Debug("timer: remove item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index)
}
return
}
// InitTCP listen all tcp.bind and start accept connections.
func InitTCP(addrs []string, accept int) (err error) {
var (
bind string
listener *net.TCPListener
addr *net.TCPAddr
)
for _, bind = range addrs {
if addr, err = net.ResolveTCPAddr("tcp4", bind); err != nil {
log.Error("net.ResolveTCPAddr(\"tcp4\", \"%s\") error(%v)", bind, err)
return
}
if listener, err = net.ListenTCP("tcp4", addr); err != nil {
log.Error("net.ListenTCP(\"tcp4\", \"%s\") error(%v)", bind, err)
return
}
if Debug {
log.Debug("start tcp listen: \"%s\"", bind)
}
// split N core accept
for i := 0; i < accept; i++ {
go acceptTCP(DefaultServer, listener)
}
}
return
}
func (server *Server) Bucket(subKey string) *Bucket {
idx := cityhash.CityHash32([]byte(subKey), uint32(len(subKey))) % server.bucketIdx
if Debug {
log.Debug("\"%s\" hit channel bucket index: %d use cityhash", subKey, idx)
}
return server.Buckets[idx]
}
// pushproc merge proto and push msgs in batch.
func (r *Room) pushproc(timer *itime.Timer, batch int, sigTime time.Duration) {
var (
n int
last time.Time
p *proto.Proto
td *itime.TimerData
buf = bytes.NewWriterSize(int(proto.MaxBodySize))
)
log.Debug("start room: %d goroutine", r.id)
td = timer.Add(sigTime, func() {
select {
case r.proto <- roomReadyProto:
default:
}
})
for {
if p = <-r.proto; p == nil {
break // exit
} else if p != roomReadyProto {
// merge buffer ignore error, always nil
p.WriteTo(buf)
if n++; n == 1 {
last = time.Now()
timer.Set(td, sigTime)
continue
} else if n < batch {
if sigTime > time.Now().Sub(last) {
continue
}
}
} else {
if n == 0 {
continue
}
}
broadcastRoomBytes(r.id, buf.Buffer())
n = 0
// TODO use reset buffer
// after push to room channel, renew a buffer, let old buffer gc
buf = bytes.NewWriterSize(buf.Size())
}
timer.Del(td)
log.Debug("room: %d goroutine exit", r.id)
}
// Push pushes the element x onto the heap. The complexity is
// O(log(n)) where n = h.Len().
func (t *Timer) add(td *TimerData) {
var d itime.Duration
td.index = len(t.timers)
// add to the minheap last node
t.timers = append(t.timers, td)
t.up(td.index)
if td.index == 0 {
// if first node, signal start goroutine
d = td.Delay()
t.signal.Reset(d)
if Debug {
log.Debug("timer: add reset delay %d ms", int64(d)/int64(itime.Millisecond))
}
}
if Debug {
log.Debug("timer: push item key: %s, expire: %s, index: %d", td.Key, td.ExpireString(), td.index)
}
return
}
func (this *RoomBucket) Get(roomId int32) (r *Room) {
this.rwLock.RLock()
room, ok := this.rooms[roomId]
if !ok {
room = NewRoom(roomId, this.round.Timer(this.roomNum), this.options)
this.rooms[roomId] = room
this.roomNum++
log.Debug("new roomId:%d num:%d", roomId, this.roomNum)
}
this.rwLock.RUnlock()
return room
}
// expire removes the minimum element (according to Less) from the heap.
// The complexity is O(log(n)) where n = max.
// It is equivalent to Del(0).
func (t *Timer) expire() {
var (
fn func()
td *TimerData
d itime.Duration
)
t.lock.Lock()
for {
if len(t.timers) == 0 {
d = infiniteDuration
if Debug {
log.Debug("timer: no other instance")
}
break
}
td = t.timers[0]
if d = td.Delay(); d > 0 {
break
}
fn = td.fn
// let caller put back
t.del(td)
t.lock.Unlock()
if fn == nil {
log.Warn("expire timer no fn")
} else {
if Debug {
log.Debug("timer key: %s, expire: %s, index: %d expired, call fn", td.Key, td.ExpireString(), td.index)
}
fn()
}
t.lock.Lock()
}
t.signal.Reset(d)
if Debug {
log.Debug("timer: expier reset delay %d ms", int64(d)/int64(itime.Millisecond))
}
t.lock.Unlock()
return
}
func serveTCP(server *Server, conn *net.TCPConn, r int) {
var (
// timer
tr = server.round.Timer(r)
rp = server.round.Reader(r)
wp = server.round.Writer(r)
// ip addr
lAddr = conn.LocalAddr().String()
rAddr = conn.RemoteAddr().String()
)
if Debug {
log.Debug("start tcp serve \"%s\" with \"%s\"", lAddr, rAddr)
}
server.serveTCP(conn, rp, wp, tr)
}
// TODO linger close?
func (server *Server) serveTCP(conn *net.TCPConn, rp, wp *bytes.Pool, tr *itime.Timer) {
var (
err error
key string
hb time.Duration // heartbeat
p *proto.Proto
b *Bucket
trd *itime.TimerData
rb = rp.Get()
wb = wp.Get()
ch = NewChannel(server.Options.CliProto, server.Options.SvrProto, define.NoRoom)
rr = &ch.Reader
wr = &ch.Writer
)
ch.Reader.ResetBuffer(conn, rb.Bytes())
ch.Writer.ResetBuffer(conn, wb.Bytes())
// handshake
trd = tr.Add(server.Options.HandshakeTimeout, func() {
conn.Close()
})
// must not setadv, only used in auth
if p, err = ch.CliProto.Set(); err == nil {
if key, ch.RoomId, hb, err = server.authTCP(rr, wr, p); err == nil {
b = server.Bucket(key)
err = b.Put(key, ch, tr)
}
}
if err != nil {
conn.Close()
rp.Put(rb)
wp.Put(wb)
tr.Del(trd)
log.Error("key: %s handshake failed error(%v)", key, err)
return
}
trd.Key = key
tr.Set(trd, hb)
// hanshake ok start dispatch goroutine
go server.dispatchTCP(key, conn, wr, wp, wb, ch)
for {
if p, err = ch.CliProto.Set(); err != nil {
break
}
if err = p.ReadTCP(rr); err != nil {
break
}
//p.Time = *globalNowTime
if p.Operation == define.OP_HEARTBEAT {
tr.Set(trd, hb)
p.Body = nil
p.Operation = define.OP_HEARTBEAT_REPLY
if Debug {
log.Debug("key: %s receive heartbeat", key)
}
} else {
if err = server.operator.Operate(p); err != nil {
break
}
}
ch.CliProto.SetAdv()
ch.Signal()
}
if err != nil && err != io.EOF {
log.Error("key: %s server tcp failed error(%v)", key, err)
}
b.Del(key)
tr.Del(trd)
rp.Put(rb)
conn.Close()
ch.Close()
if err = server.operator.Disconnect(key, ch.RoomId); err != nil {
log.Error("key: %s operator do disconnect error(%v)", key, err)
}
if Debug {
log.Debug("key: %s server tcp goroutine exit", key)
}
return
}
// dispatch accepts connections on the listener and serves requests
// for each incoming connection. dispatch blocks; the caller typically
// invokes it in a go statement.
func (server *Server) dispatchTCP(key string, conn *net.TCPConn, wr *bufio.Writer, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
var (
p *proto.Proto
err error
)
if Debug {
log.Debug("key: %s start dispatch tcp goroutine", key)
}
for {
p = ch.Ready()
if Debug {
log.Debug("key:%s dispatch msg:%v", key, *p)
}
switch p {
case proto.ProtoFinish:
if Debug {
log.Debug("key: %s wakeup exit dispatch goroutine", key)
}
goto failed
case proto.ProtoReady:
// fetch message from svrbox(client send)
for {
if p, err = ch.CliProto.Get(); err != nil {
err = nil // must be empty error
break
}
//LogSlow(SlowLogTypeReceive, key, p)
if err = p.WriteTCP(wr); err != nil {
goto failed
}
p.Body = nil // avoid memory leak
ch.CliProto.GetAdv()
}
default:
// server send
//LogSlow(SlowLogTypeReceive, key, p)
if err = p.WriteTCP(wr); err != nil {
goto failed
}
}
// only hungry flush response
if err = wr.Flush(); err != nil {
break
}
}
failed:
if err != nil {
log.Error("key: %s dispatch tcp error(%v)", key, err)
}
conn.Close()
wp.Put(wb)
// must ensure all channel message discard, for reader won't blocking Signal
for {
if p == proto.ProtoFinish {
break
}
p = ch.Ready()
}
if Debug {
log.Debug("key: %s dispatch goroutine exit", key)
}
return
}
func websocketReadProto(conn *websocket.Conn, p *Proto) error {
msg, _ := json.Marshal(p)
log.Debug("%s", string(msg))
return websocket.JSON.Receive(conn, p)
}
func initWebsocket() {
origin := "http://" + Conf.WebsocketAddr + "/sub"
url := "ws://" + Conf.WebsocketAddr + "/sub"
conn, err := websocket.Dial(url, "", origin)
if err != nil {
log.Error("websocket.Dial(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
seqId := int32(0)
proto.SeqId = seqId
proto.Body = []byte("{\"test\":1}")
if err = websocketWriteProto(conn, proto); err != nil {
log.Error("websocketWriteProto() error(%v)", err)
return
}
if err = websocketReadProto(conn, proto); err != nil {
log.Error("websocketReadProto() error(%v)", err)
return
}
log.Debug("auth ok, proto: %v", proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
// test heartbeat
//time.Sleep(time.Second * 31)
seqId++
// op_test
proto1.Operation = OP_TEST
proto1.SeqId = seqId
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
seqId++
time.Sleep(10000 * time.Millisecond)
}
}()
// reader
for {
if err = websocketReadProto(conn, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("receive heartbeat")
if err = conn.SetReadDeadline(time.Now().Add(25 * time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Debug("body: %s", string(proto.Body))
}
}
}
func startClient(key string) {
//time.Sleep(time.Duration(mrand.Intn(30)) * time.Second)
quit := make(chan bool, 1)
defer close(quit)
conn, err := net.Dial("tcp", os.Args[3])
if err != nil {
log.Error("net.Dial(\"%s\") error(%v)", os.Args[3], err)
return
}
seqId := int32(0)
wr := bufio.NewWriter(conn)
rd := bufio.NewReader(conn)
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
proto.SeqId = seqId
proto.Body = []byte(key)
if err = tcpWriteProto(wr, proto); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
if err = tcpReadProto(rd, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
log.Debug("key:%s auth ok, proto: %v", key, proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = tcpWriteProto(wr, proto1); err != nil {
log.Error("key:%s tcpWriteProto() error(%v)", key, err)
return
}
log.Debug("key:%s Write heartbeat", key)
// test heartbeat
time.Sleep(heart)
seqId++
select {
case <-quit:
return
default:
}
}
}()
// reader
for {
if err = tcpReadProto(rd, proto); err != nil {
log.Error("key:%s tcpReadProto() error(%v)", key, err)
quit <- true
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("key:%s receive heartbeat", key)
if err = conn.SetReadDeadline(time.Now().Add(heart + 60*time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
quit <- true
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Info("key:%s msg: %s", key, string(proto.Body))
atomic.AddInt64(&countDown, 1)
}
}
}
func initWebsocketTLS() {
origin := "https://" + Conf.WebsocketAddr + "/sub"
url := "wss://" + Conf.WebsocketAddr + "/sub"
conf, err := websocket.NewConfig(url, origin)
if err != nil {
log.Error("websocket.NewConfig(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
roots := x509.NewCertPool()
certPem, err := ioutil.ReadFile(Conf.CertFile)
if err != nil {
panic(err)
}
ok := roots.AppendCertsFromPEM(certPem)
if !ok {
panic("failed to parse root certificate")
}
tlsConf := &tls.Config{
//InsecureSkipVerify: true,
RootCAs: roots,
ServerName: "bili.com",
}
conf.TlsConfig = tlsConf
conn, err := websocket.DialConfig(conf)
if err != nil {
log.Error("websocket.Dial(\"%s\") error(%v)", Conf.WebsocketAddr, err)
return
}
proto := new(Proto)
proto.Ver = 1
// auth
// test handshake timeout
// time.Sleep(time.Second * 31)
proto.Operation = OP_AUTH
seqId := int32(0)
proto.SeqId = seqId
proto.Body = []byte("{\"test\":1}")
if err = websocketWriteProto(conn, proto); err != nil {
log.Error("websocketWriteProto() error(%v)", err)
return
}
if err = websocketReadProto(conn, proto); err != nil {
log.Error("websocketReadProto() error(%v)", err)
return
}
log.Debug("auth ok, proto: %v", proto)
seqId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT
proto1.SeqId = seqId
proto1.Body = nil
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
// test heartbeat
//time.Sleep(time.Second * 31)
seqId++
// op_test
proto1.Operation = OP_TEST
proto1.SeqId = seqId
if err = websocketWriteProto(conn, proto1); err != nil {
log.Error("tcpWriteProto() error(%v)", err)
return
}
seqId++
time.Sleep(10000 * time.Millisecond)
}
}()
// reader
for {
if err = websocketReadProto(conn, proto); err != nil {
log.Error("tcpReadProto() error(%v)", err)
return
}
if proto.Operation == OP_HEARTBEAT_REPLY {
log.Debug("receive heartbeat")
if err = conn.SetReadDeadline(time.Now().Add(25 * time.Second)); err != nil {
log.Error("conn.SetReadDeadline() error(%v)", err)
return
}
} else if proto.Operation == OP_TEST_REPLY {
log.Debug("body: %s", string(proto.Body))
} else if proto.Operation == OP_SEND_SMS_REPLY {
log.Debug("body: %s", string(proto.Body))
}
}
}