func (t *TcpReceiver) clearFootPrint(wg *sync.WaitGroup) { defer wg.Done() ch1 := make(chan bool) //用于安全退出 ch2 := time.After(time.Hour) //用于定时任务 go lib.HandleQuitSignal(func() { ch1 <- true }) loop: for { select { //监听一个chan以便安全退出 case <-ch1: break loop case <-ch2: //若这个case未执行完,而ch1已可读,select会保证这个case执行完 now := time.Now().Unix() t.mutex.Lock() for code, appear := range t.footPrint { if now-appear.Time >= 86400 { delete(t.footPrint, code) } } t.saveFootPrint() t.mutex.Unlock() ch2 = time.After(time.Hour) //用于定时任务 } } loglib.Info("clear footprint quit!") }
func (t *TcpReceiver) Start() { tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress) lib.CheckError(err) listener, err := net.ListenTCP("tcp", tcpAddr) lib.CheckError(err) wg := &sync.WaitGroup{} wg.Add(1) go t.clearFootPrint(wg) //主routine信号处理 go lib.HandleQuitSignal(func() { //接收到信号关闭listenner,此时Accept会马上返回一个nil 的conn listener.Close() loglib.Info("close tcp receiver's listener.") }) defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err)) } loglib.Info("wait connections finish...") wg.Wait() loglib.Info("all connections have been processed. quit.") close(t.buffer) //关闭chan t.saveFootPrint() t.wq.AllDone() }() for { conn, err := listener.Accept() if conn == nil { break } lib.CheckError(err) wg.Add(1) go t.handleConnnection(conn, wg) } }
func (this *HeartBeat) Run() { i := 0 //尝试注册3次 for i < 3 && !this.registerSelf(true) { i++ time.Sleep(10 * time.Second) } l, err := net.Listen("tcp", ":"+this.port) if err != nil { loglib.Error("heart beat " + err.Error()) return } defer l.Close() go lib.HandleQuitSignal(func() { this.registerSelf(false) l.Close() loglib.Info("close heart beat listener.") }) //heart beat 不是太重要,所以退出处理比较简单 for { conn, err := l.Accept() if conn == nil { //listener关闭 break } if err != nil { loglib.Error("heart beat " + err.Error()) break } go this.handleConnection(conn) } this.wq.AllDone() }
//goroutine func (s *Sender) Start() { // conn := s.getConnection() //初始化fileCacheList once.Do(s.reloadFileCache) //收尾工作 defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("sender %d panic:%v", s.id, err)) } s.saveBufferInChan() //s.saveMemCache() s.connection.close() s.wq.AllDone() }() go s.pickPacks() //var connLost = 0 var quit = false go lib.HandleQuitSignal(func() { quit = true s.connection.close() }) var sendInterval = time.Duration(2000) //间隔稍大,避免发送文件缓存时因无连接或其他错误进入死循环 var timeoutChan = time.After(sendInterval * time.Millisecond) for !quit { select { case b := <-s.memBuffer: //send b result := s.sendBuffer(b) if result == false { //改为直接放入文件缓存 s.writeToFile(b) } case <-timeoutChan: timeoutChan = time.After(sendInterval * time.Millisecond) // send from file e := fileList.Remove() if e != nil { // file list is not empty filename := e.Value.(string) // fmt.Println("sender ",s.id,": get file :",filename) data, err := ioutil.ReadFile(filename) if err != nil { // fmt.Println("sender ",s.id,":",err) if _, ok := err.(*os.PathError); !ok { fileList.PushBack(filename) } loglib.Error(fmt.Sprintf("sender%d read file cache %s error:%s", s.id, filename, err.Error())) } else { packId := tcp_pack.GetPackId(data) //debug info loglib.Info(fmt.Sprintf("sender%d read pack %s from file: %s, len: %d", s.id, packId, filename, len(data))) //debug info result := s.sendData2(data) if result == true { // s.fileCacheList.Remove(front) // log.Println("sender ",s.id,":removed file:",filename, "for pack", packId)//debug info err = os.Remove(filename) lib.CheckError(err) timeoutChan = time.After(time.Millisecond) //发送成功,不用再等待 } else { fileList.PushBack(filename) // fmt.Println("sender ",s.id,": pushback file :",filename) } } } } } }
func (this *Tailler) taillingCurrent(receiveChan chan map[string]string) { var n_lines = "" if this.lineNum >= 0 { n_lines = fmt.Sprintf("+%d", this.lineNum+1) //略过已经tail过的行 } else { n_lines = "0" //从最后开始 } var quit = false //收尾工作 defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("tailler panic:%v", err)) } //如果是quit,丢弃不完整的包 if quit { this.lineNum -= this.lineNum % this.recvBufSize } saveLineRecord(this.recordPath, this.currFile, this.lineNum) this.wq.AllDone() }() //启动时读取行号,以后都从首行开始 cmd := exec.Command("tail", "-F", "-n", n_lines, this.currFile) n_lines = "+1" stdout, err := cmd.StdoutPipe() if err != nil { loglib.Error("open pipe error") } //系统信号监听 go lib.HandleQuitSignal(func() { quit = true if cmd.Process != nil { cmd.Process.Kill() //关闭tail命令,不然读取循环无法终止 } }) //日志切割检测 go func() { nextHour := this.fileHour.Add(time.Hour) nextHourFile := this.getLogFileByTime(nextHour) timeToWait := 10 * time.Minute //到达下一小时后,等待日志文件的最长时间,10分钟 for { if quit { break } if lib.FileExists(nextHourFile) || time.Now().Sub(nextHour) > timeToWait { currFile := this.currFile totalLines := this.GetTotalLines(currFile) loglib.Info(fmt.Sprintf("log rotated! previous file: %s, total lines: %d", currFile, totalLines)) //在kill前进行文件切换,避免kill后新的tail启动时文件名还是旧的 this.fileHour = nextHour this.currFile = nextHourFile nextHour = nextHour.Add(time.Hour) nextHourFile = this.getLogFileByTime(nextHour) //发现日志切割,等待1分钟 i := 0 done := false for { if this.lineNum >= totalLines { done = true } if done || i > 60 { if cmd.Process != nil { cmd.Process.Kill() //关闭tail命令,不然读取循环无法终止 } if done { loglib.Info("finish tail " + currFile) } else { loglib.Info("tail " + currFile + " timeout") } break } i++ time.Sleep(time.Second) } } time.Sleep(time.Second) } }() outer: for { currFile := this.currFile //缓存当前tail的文件名 hourStr := this.fileHour.Format(this.hourStrFmt) cmd.Start() loglib.Info("begin current log: " + currFile) rd := bufio.NewReader(stdout) for line, err := rd.ReadString('\n'); err == nil; line, err = rd.ReadString('\n') { //fmt.Print(line) if quit { break outer } this.lineNum++ m := map[string]string{"hour": hourStr, "line": line} receiveChan <- m if this.lineNum%this.recvBufSize == 0 { saveLineRecord(this.recordPath, currFile, this.lineNum) } } if err := cmd.Wait(); err != nil { loglib.Info("wait sys tail error!" + err.Error()) } loglib.Info(fmt.Sprintf("%s tailed %d lines", currFile, this.lineNum)) if quit { break } // 完整tail一个文件 m := map[string]string{"hour": hourStr, "line": changeStr} receiveChan <- m saveLineRecord(this.recordPath, currFile, this.lineNum) //begin a new file this.lineNum = 0 cmd = exec.Command("tail", "-F", "-n", n_lines, this.currFile) stdout, err = cmd.StdoutPipe() if err != nil { loglib.Error("open pipe error") break } } }
func (this *Tailler) taillingPrevious(filePath string, lineNum int, hourStr string, receiveChan chan map[string]string) bool { var n_lines = "" if lineNum >= 0 { n_lines = fmt.Sprintf("+%d", lineNum+1) //略过已经tail过的行 } else { n_lines = "0" //从最后开始 } loglib.Info("begin previous log: " + filePath + " from line: " + n_lines) var quit = false //收尾工作 defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("tailler panic:%v", err)) } //如果是quit,丢弃不完整的包 if quit { lineNum -= lineNum % this.recvBufSize } saveLineRecord(this.recordPath, filePath, lineNum) }() //启动时读取行号,以后都从首行开始 cmd := exec.Command("tail", "-n", n_lines, filePath) stdout, err := cmd.StdoutPipe() if err != nil { loglib.Error("open pipe error") } //系统信号监听 go lib.HandleQuitSignal(func() { quit = true if cmd.Process != nil { cmd.Process.Kill() //关闭tail命令,不然读取循环无法终止 } }) cmd.Start() rd := bufio.NewReader(stdout) for line, err := rd.ReadString('\n'); err == nil; line, err = rd.ReadString('\n') { //fmt.Print(line) if quit { break } lineNum++ m := map[string]string{"hour": hourStr, "line": line} receiveChan <- m if lineNum%this.recvBufSize == 0 { saveLineRecord(this.recordPath, filePath, lineNum) } } if err := cmd.Wait(); err != nil { loglib.Info("wait sys tail error!" + err.Error()) } loglib.Info(fmt.Sprintf("%s tailed %d lines", filePath, lineNum)) if !quit { // 完整tail一个文件 m := map[string]string{"hour": hourStr, "line": changeStr} receiveChan <- m saveLineRecord(this.recordPath, filePath, lineNum) } return quit }
func (t *TcpReceiver) handleConnnection(conn net.Conn, wg *sync.WaitGroup) { defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("tcp receiver connection panic:%v", err)) } conn.Close() wg.Done() }() /* 用于标识收到退出信号后,能否直接退出 只要接收信号时,包没有收完,都是可退出的, 发送方会缓存以后重传; 如果收完了就不能直接退出,可能包已传给下一级处理但是 却告诉发送方发送失败 */ var quit = false //用于标识是否要退出 go lib.HandleQuitSignal(func() { //关闭连接,避免阻塞在网络io上 conn.Close() quit = true }) request := make([]byte, 512*1024) //缓冲为512k var packLen int = 0 currLen := 0 var b = new(bytes.Buffer) var content = new(bytes.Buffer) inAddr := conn.RemoteAddr().String() parts := strings.Split(inAddr, ":") inIp := parts[0] packId := "unkown" var routeInfo map[string]string var rePull = false //是否补拉,如果是补拉就不做重复包检验 loglib.Info("incoming: " + inAddr) outer: for !quit { st := time.Now() if packLen == 0 { conn.SetReadDeadline(time.Now().Add(5 * time.Minute)) time1 := time.Now() //时间打点 // read zlib pack header length buf := make([]byte, 4) _, err := conn.Read(buf) if err != nil { loglib.Warning(fmt.Sprintf("conn:%s, get header len, tcp receiver read error:%s, elapse:%s", inAddr, err.Error(), time.Now().Sub(time1))) break } l, _ := binary.Uvarint(buf) headerLen := int(l) //get pack header headerBuf := make([]byte, headerLen) time2 := time.Now() _, err = conn.Read(headerBuf) if err != nil { loglib.Warning(fmt.Sprintf("conn:%s, get header, tcp receiver read error:%s, elapse:%s", inAddr, err.Error(), time.Now().Sub(time2))) break } //是否补拉 route0 := tcp_pack.ParseHeader(headerBuf) if v, ok := route0["repull"]; ok && v == "1" { rePull = true } else { rePull = false } buf = append(buf, headerBuf...) header, _, err := tcp_pack.ExtractHeader(buf) if err != nil { loglib.Error("wrong format header " + string(headerBuf) + " " + err.Error()) conn.Write([]byte("wrong header")) break } packId = tcp_pack.GetPackId(buf) packLen = header.PackLen currLen = 0 routeInfo = make(map[string]string) b = new(bytes.Buffer) content = new(bytes.Buffer) loglib.Info(fmt.Sprintf("conn:%s, start receive pack %s, pack len:%d, header len:%d, header elapse:%s", inAddr, packId, packLen, headerLen, time.Now().Sub(time1))) b.Write(buf) routeInfo["ip"] = lib.GetIp() routeInfo["stage"] = "tcp recv" routeInfo["st"] = st.Format("2006-01-02 15:04:05.000") } //读包体的超时 conn.SetReadDeadline(time.Now().Add(5 * time.Minute)) time3 := time.Now() //read enough bytes for currLen < packLen { requestLen, err := conn.Read(request) if requestLen == 0 || err != nil { //sender有重发机制,所以可丢弃 packLen = 0 //设为0以便读取新的包 ed := time.Now() loglib.Warning(fmt.Sprintf("conn:%s, not full! ip:%s, packid:%s, received:%d, end recv:%s, elapse:%s, body elapse:%s, error:%s", inAddr, inIp, packId, currLen, ed, ed.Sub(st), ed.Sub(time3), err.Error())) break outer //连接出错直接跳出外层循环 } currLen += requestLen content.Write(request[:requestLen]) } if packLen > 0 && currLen >= packLen { //收完马上应答 _, err := conn.Write([]byte("ok")) if err != nil { loglib.Warning(fmt.Sprintf("ip:%s, packid:%s received, but response back error:%s", inIp, packId, err.Error())) } else { loglib.Info(fmt.Sprintf("conn:%s, response to packid:%s", inAddr, packId)) } //避免收到重复包(补拉例外) appeared, ok, code := t.hasAppeared(content) if !ok || rePull { ed := time.Now() routeInfo["ed"] = ed.Format("2006-01-02 15:04:05.000") routeInfo["elapse"] = ed.Sub(st).String() b.Write(content.Bytes()) vbytes := tcp_pack.Packing(b.Bytes(), routeInfo, true) b = bytes.NewBuffer(vbytes) t.buffer <- *b packAppear := PackAppear{time.Now().Unix(), packId} t.mutex.Lock() t.footPrint[code] = packAppear //这里挂过 t.mutex.Unlock() loglib.Info(fmt.Sprintf("conn:%s, finish ip:%s, packid:%s, repull:%v, received:%d, elapse:%s, body elapse:%s", inAddr, inIp, packId, rePull, currLen, ed.Sub(st), ed.Sub(time3))) } else { loglib.Info(fmt.Sprintf("conn:%s, pack %s repeat %s already appear at %s", inAddr, packId, appeared.Id, time.Unix(appeared.Time, 0))) } packLen = 0 } } loglib.Info("conn finish: " + inAddr) }
//重新保存先前失败的文档 func (this *MongoDbOutputer) retrySave(wg *sync.WaitGroup, routineId int) { var session *mgo.Session var coll *mgo.Collection //routine一般都要copy或clone session,clone能保证一致性 if this.session != nil { session = this.session.Clone() } defer func() { if session != nil { session.Close() } wg.Done() loglib.Info(fmt.Sprintf("mongodb outputer retry routine %d quit.", routineId)) }() var quit = false go lib.HandleQuitSignal(func() { quit = true }) dateStr := "" loglib.Info(fmt.Sprintf("mongodb outputer retry routine %d start", routineId)) for !quit { e := this.fileList.Remove() if e != nil { filename := e.Value.(string) b, err := ioutil.ReadFile(filename) if err != nil { if _, ok := err.(*os.PathError); !ok { this.fileList.PushBack(filename) //非路径错误,下次再试 } loglib.Error(fmt.Sprintf("load cache %s error:%v", filename, err)) } else { m := bson.M{} err = json.Unmarshal(b, &m) if err != nil { loglib.Error(fmt.Sprintf("unmarshar %s error:%v", filename, err)) } else { tp, _ := m["type"].(string) date, _ := m["date"].(string) if date != dateStr && session != nil { coll = session.DB(this.db + date).C(this.collection) //按天分库 dateStr = date } if tp == "bulk" { data, _ := m["data"].([]interface{}) err = this.bulkSaveBson(coll, data...) } else { data, _ := m["data"].(map[string]interface{}) sel := bson.M{this.transactionIdKey: data[this.transactionIdKey]} up := bson.M{"$set": data} _, err = this.upsertBson(coll, sel, up) } if err != nil { this.fileList.PushBack(filename) loglib.Error(fmt.Sprintf("re-save cache %s error:%v", filename, err)) if session.Ping() != nil { //refresh go-routine's session if possible this.reCloneRoutineSession(&session) if session.Ping() == nil { loglib.Info(fmt.Sprintf("retry routine %d re-conn", routineId)) } } } else { err = os.Remove(filename) if err != nil { loglib.Error(fmt.Sprintf("remove file: %s error:%v", filename, err)) } else { loglib.Info(fmt.Sprintf("cache file: %s send out", filename)) } } } } } time.Sleep(500 * time.Millisecond) } }