func main() {
log_ := "NOTICE: 15-05-25 15:32:26 errno[0] client[10.77.141.87] uri[/commoncomponent?header=1&footer=1&menu=1&hlflag=child] user[15000000000209285] refer[] cookie[SESSIONID=02bef8adbvrquan5j1gjcsd9u3;U_UID=02bef8adbvrquan5j1gjcsd9u3;tempNoticeClosed=1;CITY_ID=110100;uid=15000000000209285;PHPSESSID=02bef8adbvrquan5j1gjcsd9u3] post[] ts[0.045583009719849]\n"
var err error
if checkFileIsExist(options.log) {
log.Info("Log file: %s", options.log)
} else {
_, err = os.Create(options.log)
if err != nil {
log.Error("file not exists")
}
os.Exit(1)
}
waitGroup := make(chan interface{})
for i := 0; i < options.concurrency; i++ {
go func() {
f, err := os.OpenFile(options.log, os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
log.Error(err)
}
for j := 0; j < options.last; j++ {
for k := 0; k < options.requests; k++ {
appendLog(f, log_)
}
time.Sleep(time.Second)
}
waitGroup <- true
}()
}
for i := 0; i < options.concurrency; i++ {
<-waitGroup
}
}
func (this *Protocol) Read() ([]byte, []byte, error) {
buf := make([]byte, HEAD_LENGTH)
err := this.ReadN(this.Conn, buf, HEAD_LENGTH)
if err != nil {
log.Error("[Protocol] Read data length error: %s", err.Error())
return []byte{}, []byte{}, err
}
//data length
b_buf := bytes.NewBuffer(buf[:4])
var dataLength int32
binary.Read(b_buf, binary.BigEndian, &dataLength)
//app length
var appLength int32
b_buf.Write(buf[4:8])
binary.Read(b_buf, binary.BigEndian, &appLength)
//app + data
payloadLength := int(dataLength + appLength)
payload := make([]byte, payloadLength)
err = this.ReadN(this.Conn, payload, payloadLength)
if err != nil && err != io.EOF {
log.Error("[Protocol] Read data error: %s", err.Error())
return []byte{}, []byte{}, err
}
return payload[:appLength], payload[appLength:payloadLength], nil
}
// TODO more secure token generator
func getToken() (token string) {
token = str.Rand(32)
err := db.MgoSession().DB("pushd").C("token").Insert(bson.M{"tk": token, "expire": time.Now()})
if err != nil {
log.Error("generate token error: %s", err.Error())
return ""
}
return
}
func Serv() {
if config.PushdConf.MsgFlushPolicy == config.MSG_FLUSH_EVERY_SECOND {
go func() {
for {
select {
case <-time.Tick(time.Second):
//get current buffer length
cLen := len(writeBuffer)
if cLen > 0 {
records := make([]*MsgTuple, 0)
for i := 0; i < cLen; i++ {
if mt, ok := <-writeBuffer; ok {
records = append(records, mt)
} else {
// Ghost appears...
log.Error("Msg gone away...")
break
}
}
err := driver.storeMulti(records)
if err != nil {
log.Error("Multi store msg log error: %s", err.Error())
}
}
}
}
}()
}
for {
select {
case mt := <-msgQueue:
if config.PushdConf.MsgFlushPolicy == config.MSG_FLUSH_EVERY_TRX {
err := driver.store(mt)
if err != nil {
log.Error("Store msg log error: %s", err.Error())
}
} else {
writeBuffer <- mt
}
}
}
}
func (this *PushdClient) Connect() error {
var err error
this.Conn, err = net.Dial("tcp", this.serverAddr)
if err != nil {
log.Error("Connect server[%s] error: %s", this.serverAddr, err.Error())
this.Conn = nil
return err
}
this.proto.SetConn(this.Conn)
return nil
}
func (this *Poller) watchDir(dir string, followDir bool) {
watcher, err := fsnotify.NewWatcher()
err = watcher.Watch(dir)
if err != nil {
panic(err)
}
defer func() {
err := watcher.Close()
if err != nil {
panic(err)
}
}()
// Process events
for {
select {
case ev := <-watcher.Event:
if ev.IsCreate() {
finfo, err := os.Stat(ev.Name)
if err != nil {
log.Error(err)
continue
}
if finfo.IsDir() {
if followDir {
this.tailFilesInDir(ev.Name, "*")
} else {
continue
}
} else {
go this.tailFile(ev.Name, true)
}
}
case err := <-watcher.Error:
log.Error("error:", err)
}
}
}
//Auth for client
func authClient(token string) (string, error) {
var result interface{}
db.MgoSession().SetMode(mgo.Monotonic, true)
_, err := db.MgoSession().DB("pushd").C("token").Find(bson.M{"tk": token}).Select(bson.M{"_id": 0, "tk": 1}).Apply(mgo.Change{Remove: true}, &result)
if err == mgo.ErrNotFound {
return "", errors.New("Client auth fail")
} else if err != nil {
log.Error("get token from db error: %s", err.Error())
return "", errors.New("Client auth fail")
}
return "Client auth succeed", nil
}
func (this *MongoStorage) fetchByChannelAndTs(channel string, ts int64) (result []interface{}, err error) {
if channel == "" {
return nil, errors.New("No channel specified")
}
err = db.MgoSession().DB("pushd").C("msg_log").Find(bson.M{"ts": bson.M{"$gte": ts}, "channel": channel}).Select(bson.M{"_id": 0}).All(&result)
if err != nil && err != mgo.ErrNotFound {
log.Error("fetch messages log from db error: %s", err.Error())
}
if len(result) == 0 {
return []interface{}{}, nil
}
return
}
func authServer(appId, secretKey string) (string, error) {
c := db.MgoSession().DB("pushd").C("user")
var result interface{}
err := c.Find(bson.M{"appId": appId}).One(&result)
if err == mgo.ErrNotFound {
return "", errors.New("Server auth fail")
} else if err != nil {
log.Error("Error occured when query mongodb: %s", err.Error())
}
key := result.(bson.M)["secretKey"]
if key == secretKey {
return "Server auth succeed", nil
}
return "", errors.New("Server auth fail")
}
func (this *Poller) tailFilesInDirRecursive(dir string) {
dir_list, err := ioutil.ReadDir(dir)
if err != nil {
log.Error("read dir error: %s", err.Error())
return
}
for _, path := range dir_list {
fullPath := fmt.Sprintf("%s%s%s", dir, PATH_SEP, path.Name())
if path.IsDir() == true {
this.tailFilesInDirRecursive(fullPath)
} else {
go this.tailFile(fullPath, false)
}
}
go this.watchDir(dir, true)
}
func (this *Poller) tailFile(filename string, isNew bool) {
log.Info("Tail file: %s", filename)
var location *tail.SeekInfo
if isNew {
location = &tail.SeekInfo{Offset: 0, Whence: os.SEEK_SET}
} else {
location = &tail.SeekInfo{Offset: 0, Whence: os.SEEK_END}
}
t, err := tail.TailFile(filename, tail.Config{Follow: true, Location: location})
if err != nil {
log.Error("tail file[%s] errer: %s", filename, err.Error())
}
for line := range t.Lines {
txt := line.Text
log.Info(txt)
if tag := this.filter(txt); tag != "" {
this.forwarder.Enqueue(fmt.Sprintf("%s|%s", tag, txt))
}
}
}
func shutdown() {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
debug.PrintStack()
}
}()
log.Info("Terminated")
if config.PushdConf.IsDistMode() {
err := engine.UnregisterEtc()
if err != nil {
log.Error(err)
}
}
if config.PushdConf.IsDistMode() {
s2sServer.StopTcpServer()
}
pushdServer.StopTcpServer()
time.Sleep(time.Second * 2)
os.Exit(0)
}
func (this *Forwarder) reconnect() {
if this.Conn != nil {
this.Conn.Close()
}
conn, err := net.Dial("tcp", this.config.ToAddr)
if err != nil {
log.Error(err)
}
this.Conn = conn
if conn != nil {
go func() {
for {
_, err := this.Conn.Read(make([]byte, 1000))
if err != nil {
log.Warn(err)
}
this.Conn.Close()
break
}
}()
}
}
func (this *Poller) tailFilesInDir(dir string, filenameReg string) {
sourceReg := fmt.Sprintf("^%s$", strings.Replace(filenameReg, "*", ".*?", -1))
dir_list, err := ioutil.ReadDir(dir)
if err != nil {
log.Error("read dir error: %s", err.Error())
return
}
reg := regexp.MustCompile(sourceReg)
for _, path := range dir_list {
if path.IsDir() == true {
continue
}
if matchFile(reg, path.Name()) {
go this.tailFile(fmt.Sprintf("%s%s%s", dir, PATH_SEP, path.Name()), false)
}
}
go this.watchDir(dir, false)
}
func (this *Forwarder) Send() {
for line := range this.queue {
data := this.proto.Marshal([]byte(line))
if this.Conn == nil {
this.reconnect()
}
if this.Conn != nil {
_, err := this.Write(data)
if err != nil {
log.Error("write error: %s", err.Error())
// retry
for i := 0; i < 3; i++ {
this.reconnect()
if this.Conn != nil {
_, err = this.Write(data)
if err == nil {
break
}
}
}
}
}
}
}
func appendLog(f *os.File, log_ string) {
_, err := io.WriteString(f, log_)
if err != nil {
log.Error(err)
}
}
func (this *PushdLongPollingServer) ServeSubscribe(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
channel := vars["channel"]
tsStr := vars["ts"]
ts, err := strconv.Atoi(tsStr)
if err != nil {
io.WriteString(w, "invalid timestamp")
return
}
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError)
return
}
conn, _, err := hj.Hijack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
conn.Write([]byte("HTTP/1.1 200 OK\r\n"))
conn.Write([]byte("Access-Control-Allow-Origin: *\r\n"))
conn.Write([]byte("\r\n"))
//fetch history first
if ts != 0 {
hisRet, err := history(channel, int64(ts)+1)
for i, hisRetEle := range hisRet {
retEleBson, _ := hisRetEle.(bson.M)
retEleBson["ts"] = strconv.Itoa(int(retEleBson["ts"].(int64)))
hisRet[i] = retEleBson
}
if err != nil {
log.Error(err)
}
if len(hisRet) > 0 {
var retBytes []byte
retBytes, err = json.Marshal(hisRet)
conn.Write(retBytes)
conn.Close()
return
}
}
c := server.NewClient(conn, time.Now(), server.CONN_TYPE_LONG_POLLING, nil)
client := NewClient()
client.Client = c
subscribe(client, channel)
for {
if this.sessTimeout.Nanoseconds() > int64(0) {
client.SetReadDeadline(time.Now().Add(this.sessTimeout))
}
_, err := client.Conn.Read(make([]byte, 1460))
if err != nil {
if err == io.EOF {
log.Info("Client end polling: %s", client.Conn.RemoteAddr())
client.Close()
return
} else if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
log.Info("client[%s] read timeout", client.RemoteAddr())
client.Close()
return
} else if nerr, ok := err.(net.Error); !ok || !nerr.Temporary() {
client.Close()
return
} else {
log.Info("Unexpected error: %s", err.Error())
client.Close()
return
}
}
}
}
func (this *Cmdline) Process() (ret string, err error) {
switch this.Cmd {
case CMD_SUBSCRIBE:
if !this.Client.IsClient() {
return "", ErrNotPermit
}
if len(this.Params) < 1 || this.Params[0] == "" {
return "", errors.New("Lack sub channel")
}
ret = subscribe(this.Client, this.Params[0])
case CMD_PUBLISH:
if !this.Client.IsClient() && !this.Client.IsServer() {
return "", ErrNotPermit
}
if len(this.Params) < 2 || this.Params[1] == "" {
return "", errors.New("Publish without msg\n")
} else {
ret = publish(this.Params[0], this.Params[1], false)
}
case CMD_UNSUBSCRIBE:
if !this.Client.IsClient() {
return "", ErrNotPermit
}
if len(this.Params) < 1 || this.Params[0] == "" {
return "", errors.New("Lack unsub channel")
}
ret = unsubscribe(this.Client, this.Params[0])
case CMD_HISTORY:
if !this.Client.IsClient() {
return "", ErrNotPermit
}
if len(this.Params) < 2 {
return "", errors.New("Invalid Params for history")
}
ts, err := strconv.ParseInt(this.Params[1], 10, 64)
if err != nil {
return "", err
}
channel := this.Params[0]
hisRet, err := fullHistory(channel, ts)
if err != nil {
log.Error(err)
}
var retBytes []byte
retBytes, err = json.Marshal(hisRet)
ret = string(retBytes)
//use one appId/secretKey pair
case CMD_AUTH_SERVER:
if len(this.Params) < 2 {
return "", errors.New("Invalid Params for auth_server")
}
if this.Client.IsServer() {
ret = "Already authed server"
err = nil
} else {
ret, err = authServer(this.Params[0], this.Params[1])
if err == nil {
this.Client.SetServer()
}
}
case CMD_TOKEN:
if !this.Client.IsServer() {
return "", ErrNotPermit
}
ret = getToken()
if ret == "" {
return "", errors.New("gettoken error")
}
case CMD_AUTH_CLIENT:
if len(this.Params) < 1 {
return "", errors.New("Invalid Params for auth_client")
}
if this.Client.IsClient() {
ret = "Already authed client"
err = nil
} else {
ret, err = authClient(this.Params[0])
if err == nil {
this.Client.SetClient()
}
}
case CMD_PING:
return OUTPUT_PONG, nil
default:
return "", errors.New(fmt.Sprintf("Cmd not found: %s\n", this.Cmd))
}
return
}
