/* * 行数为非负值表示已tail的行数 * 行数为负值则都将从文件末尾开始 * 自动保存的行数只可能是bufSize的倍数或者文件总行数 */ func getLineRecord(path string) (line int, fname string) { fin, err := os.Open(path) if err != nil { _, f, l, _ := runtime.Caller(0) loglib.Error(fmt.Sprintf("%s:%d open line record `%s` error\n", f, l, path)) return -1, "" //从最后开始读 } var txt string var lineStr = "" //只读第一行 scanner := bufio.NewScanner(fin) for scanner.Scan() { txt = strings.Trim(scanner.Text(), " ") break } fin.Close() parts := strings.Split(txt, " ") if len(parts) == 2 { fname = parts[0] lineStr = parts[1] } else { lineStr = parts[0] } line, err = strconv.Atoi(lineStr) if err != nil { loglib.Error("convert line record error:" + err.Error()) line = -1 } return line, fname }
//保存footprint func (t *TcpReceiver) saveFootPrint() { vbytes, err := json.Marshal(t.footPrint) if err != nil { loglib.Error("marshal footprint error:" + err.Error()) return } err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664) if err == nil { loglib.Info("save footprint success !") } else { loglib.Error("save footprint error:" + err.Error()) } }
func saveLineRecord(path string, fname string, lineNum int) { fout, err := os.Create(path) defer fout.Close() if err != nil { loglib.Error("save line record error: " + err.Error()) return } _, err = fmt.Fprintf(fout, "%s %d", fname, lineNum) if err != nil { loglib.Error("Write line record error" + err.Error()) return } loglib.Info("save line record success!") }
func (this *MongoDbOutputer) Start() { wg := &sync.WaitGroup{} defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("mongodb outputer panic:%v", err)) } if this.session != nil { this.session.Close() } this.wq.AllDone() }() this.reloadFileCache() go this.reConnMongoDb() wg.Add(this.savers) for i := 0; i < this.savers; i++ { go this.runParse(i, wg) } nRetry := this.savers/6 + 1 wg.Add(nRetry) for i := 0; i < nRetry; i++ { go this.retrySave(wg, i) } wg.Wait() }
func (this *Tailler) Tailling(receiveChan chan map[string]string) { if this.currFile == "" { //兼容老格式,老格式无文件路径 this.currFile = this.getLogFileByTime(time.Now()) } var err error this.fileHour, err = this.getTimeFromLogName(this.currFile) if err != nil { loglib.Error("can't get time from current log file:" + this.currFile + "error:" + err.Error()) os.Exit(1) } isQuit := false for time.Since(this.fileHour).Hours() >= 1 { //说明重启时已经跟记录行号时不属于同一个小时了 isQuit = this.taillingPrevious(this.currFile, this.lineNum, this.fileHour.Format(this.hourStrFmt), receiveChan) if isQuit { break } //继续下一个小时 this.fileHour = this.fileHour.Add(time.Hour) this.currFile = this.getLogFileByTime(this.fileHour) this.lineNum = 0 } if !isQuit { //处理当前这个小时 this.taillingCurrent(receiveChan) } close(receiveChan) this.wq.AllDone() }
func (lr *LogReceiver) Run() { l, err := net.Listen("tcp", fmt.Sprintf(":%d", lr.port)) if err != nil { loglib.Error("[log receiver] " + err.Error()) return } defer l.Close() for { conn, err := l.Accept() if err != nil { loglib.Error("[log receiver] " + err.Error()) return } go lr.handleConnection(conn) } }
func (f *fileOutputer) extract(bp *bytes.Buffer) { buf := make([]byte, 4) bp.Read(buf) l, _ := binary.Uvarint(buf) headerLen := int(l) //get pack header buf = make([]byte, headerLen) bp.Read(buf) header := tcp_pack.ParseHeader(buf) r, err := zlib.NewReader(bp) if err != nil { loglib.Error("zlib reader Error: " + err.Error()) } else { lines, _ := strconv.Atoi(header["lines"]) done := false if header["done"] == "1" { done = true } f.ic.Add(header["ip"], header["hour"], header["id"], lines, done) writerKey := header["ip"] + "_" + header["hour"] fout := f.getWriter(f.writers, f.dataDir, writerKey) //一头一尾写头信息,节省硬盘 buf = append(buf, '\n') //fout.Write(buf) nn, err := io.Copy(fout, r) if err != nil { loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn)) } //fout.Write(buf) //单独存一份header便于查数 fout = f.getWriter(f.headerWriters, f.headerDir, writerKey) n, err := fout.Write(buf) if err != nil { loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error())) } if done || time.Now().Unix() > f.checkTime.Unix() { hourFinish, _ := f.ic.Check() for ip, hours := range hourFinish { for _, hour := range hours { writerKey = ip + "_" + hour } } f.closeWriters(f.writers) f.closeWriters(f.headerWriters) f.checkTime.Add(2 * time.Minute) } r.Close() } }
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear { fp := make(map[string]PackAppear) if lib.FileExists(fname) { vbytes, err := ioutil.ReadFile(fname) if err != nil { loglib.Error("read footprint file error:" + err.Error()) } else { err = json.Unmarshal(vbytes, &fp) if err != nil { loglib.Error("unmarshal footprint error:" + err.Error()) } else { loglib.Info("load footprint success !") } } } else { loglib.Warning("footprint file " + fname + " not found!") } return fp }
//init conn list from addrMap func (sc *SingleConnection) initConnection() { newConn, err := createSingleConnection(sc.currentAddr) if err != nil { loglib.Error("init err:" + err.Error()) } else { sc.conn = newConn } lib.CheckError(err) }
func (e *etlOutputer) Start() { defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("etl outputer panic:%v", err)) } e.wq.AllDone() }() spiderList, _ := e.config["spider_list"] colsFile, _ := e.config["columns_file"] hostsList, _ := e.config["hosts_white_list"] ipBlackList, _ := e.config["ip_black_list"] if colsFile != "" { e.runEtl(spiderList, colsFile, hostsList, ipBlackList) } else { loglib.Error("[error] miss columns map file!") } }
func initMongoDbSession(mongosAddr string) *mgo.Session { session, err := mgo.Dial(mongosAddr) if err != nil { loglib.Error(fmt.Sprintf("init mongodb session error:%v", err)) return nil } session.SetMode(mgo.Monotonic, true) //设置read preference session.SetSafe(&mgo.Safe{W: 2}) //设置write concern return session }
func (this *MongoDbOutputer) bulkSaveBson(coll *mgo.Collection, docs ...interface{}) (err error) { if coll != nil { err = coll.Insert(docs...) if err != nil { tmp := make([]string, 0) for _, doc := range docs { m, _ := doc.(bson.M) tid, _ := m[this.transactionIdKey].(string) tmp = append(tmp, tid) } tids := strings.Join(tmp, ",") loglib.Error(fmt.Sprintf("save %d bsons [%s] error:%v", len(docs), tids, err)) } } else { err = errors.New("bulk: collection is nil") loglib.Error(fmt.Sprintf("save bsons error:%v", err)) } return }
func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) { m, _ := selector.(bson.M) tid, _ := m[this.transactionIdKey].(string) if coll != nil { info, err = coll.Upsert(selector, doc) if err != nil { loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err)) } else { if info.Updated > 0 { loglib.Info(fmt.Sprintf("bson [%s] updated", tid)) } } } else { info = &mgo.ChangeInfo{} err = errors.New("upsert: collection is nil") loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err)) } return }
func (this *Tailler) GetTotalLines(fname string) int { cmd := exec.Command("/bin/sh", "-c", `wc -l `+fname+` | awk '{print $1}'`) out, err := cmd.Output() if err == nil { n, err := strconv.Atoi(strings.Trim(string(out), "\n")) if err != nil { loglib.Error("trans total lines " + string(out) + " error: " + err.Error()) } return n } return 0 }
func NewTailler(config map[string]string) *Tailler { val, ok := config[logFileKey] if !ok || val == "" { loglib.Error("config need log_file!") os.Exit(1) } logPath := val val, ok = config[recordFileKey] if !ok || val == "" { config[recordFileKey] = getRecordPath() } lineNum, fname := getLineRecord(config[recordFileKey]) goFmt, nLT := extractTimeFmt(logPath) if goFmt == "" { loglib.Error("log path has no time format!") os.Exit(1) } wq := lib.NewWaitQuit("tailler") bufSize, _ := strconv.Atoi(config["recv_buffer_size"]) return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq} }
func (e *etlOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File { w, ok := writers[key] if !ok || w == nil { fname := filepath.Join(parentDir, key) w1, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) writers[key] = w1 w = w1 if err != nil { loglib.Error(fmt.Sprintf("etl outputer create writer: %s error: %s", fname, err.Error())) } } return w }
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, hostsList string, ipBlackList string, wg *sync.WaitGroup) { defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("doEtl() panic:%v", err)) } wg.Done() }() loglib.Info("etl routine start") for fkey := range fkeyChan { sv := etl.NewFileSaver(colsFile, etlDir, fkey) d := etl.NewDispatcher(sv, 6, hostsList, ipBlackList) g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d) go g.Start(false) fname := filepath.Join(logDataDir, fkey) loglib.Info("start etl for " + fname) err := g.ParseFile(fname) g.Wait() // etl success // mark success if err == nil { //采用循环,增加打tag的成功率 for i := 0; i < 5; i++ { fd, err := os.Create(filepath.Join(etlDoneDir, fkey)) if err == nil { fd.Close() loglib.Info("finish etl for " + fname) break } else { loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error()) } } } else { //采用循环,增加打tag的成功率 for i := 0; i < 5; i++ { fd, err := os.Create(filepath.Join(etlFailDir, fkey)) if err == nil { fd.Close() loglib.Info("failed etl for " + fname) break } else { loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error()) } } } } loglib.Info("etl routine finish") }
func check(addr string, ch chan CheckResult) { conn, err := net.Dial("tcp", addr) if err != nil { msg := "[heart beat] connect to " + addr + " error" loglib.Error(msg) ch <- CheckResult{addr, true, msg} } else { conn.SetReadDeadline(time.Now().Add(10 * time.Second)) //读超时 var buf = make([]byte, 2) n, err := conn.Read(buf) if err != nil { msg := "[heart beat] read data from " + addr + " error" loglib.Error(msg) ch <- CheckResult{addr, true, msg} } if string(buf[:n]) == "!" { ch <- CheckResult{addr, false, ""} } else { ch <- CheckResult{addr, true, "unknow return code"} } conn.Close() } }
func New(configFile string) *Monitor { //new Monitor monitor := new(Monitor) monitor.configFile = configFile config := lib.ReadConfig(configFile) // heart beat checker cfg, ok := config["heart_beat"] if !ok { loglib.Error("miss heart beat config") return nil } checkInterval, _ := strconv.Atoi(cfg["check_interval"]) delete(cfg, "check_interval") mutex := &sync.RWMutex{} monitor.mutex = mutex monitor.checkInterval = checkInterval hbChecker := heart_beat.NewHeartBeatChecker() monitor.hbChecker = hbChecker //log receiver cfg, ok = config["receiver"] if !ok { loglib.Error("miss receiver config!") return nil } mysql := db.NewMysql(cfg["db_host"], cfg["db_port"], cfg["db_uname"], cfg["db_passwd"], cfg["db_db"], cfg["db_charset"]) monitor.dbConn = mysql monitor.ipRoleMap = getIpRoleMap(mysql) recvPort, _ := strconv.Atoi(cfg["recv_port"]) receiver := NewLogReceiver(recvPort, mysql, monitor.ipRoleMap, monitor.mutex) monitor.receiver = receiver return monitor }
func (this *HeartBeat) Run() { i := 0 //尝试注册3次 for i < 3 && !this.registerSelf(true) { i++ time.Sleep(10 * time.Second) } l, err := net.Listen("tcp", ":"+this.port) if err != nil { loglib.Error("heart beat " + err.Error()) return } defer l.Close() go lib.HandleQuitSignal(func() { this.registerSelf(false) l.Close() loglib.Info("close heart beat listener.") }) //heart beat 不是太重要,所以退出处理比较简单 for { conn, err := l.Accept() if conn == nil { //listener关闭 break } if err != nil { loglib.Error("heart beat " + err.Error()) break } go this.handleConnection(conn) } this.wq.AllDone() }
//从日志文件名获取时间,不依赖系统时间 func (this *Tailler) getTimeFromLogName(name string) (time.Time, error) { size := len(name) timePart := "" if this.nLT[0] < size && this.nLT[1] < size { timePart = name[this.nLT[0] : size-this.nLT[1]] } layout := this.goFmt loc, _ := time.LoadLocation("Local") t, err := time.ParseInLocation(layout, timePart, loc) if err != nil { loglib.Error("parse " + timePart + " against " + layout + " error:" + err.Error()) } return t, err }
//向monitor注册或取消注册,reg为true表示注册,否则是取消注册,返回true表示成功 func (this *HeartBeat) registerSelf(reg bool) bool { conn, err := lib.GetConnection(this.monitorAddr) if err != nil { return false } defer conn.Close() req := "register" if !reg { req = "unregister" } m := map[string]string{"req": req, "ip": lib.GetIp(), "port": this.port, "hostname": lib.GetHostname(), "role": this.role} msg, err := json.Marshal(m) if err != nil { loglib.Error("marshal " + req + " info error " + err.Error()) return false } _, err = conn.Write(tcp_pack.Pack(msg)) if err != nil { loglib.Error("send " + req + " info failed" + err.Error()) return false } else { plen, ret := tcp_pack.UnPack(conn) if plen > 0 { err = json.Unmarshal(ret, &m) r, ok := m["err"] if err == nil && ok && r == "0" { loglib.Info(req + " to monitor success!") return true } loglib.Error(req + " heart beat failed!") } } return false }
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) { mo.buffer = buffer mo.wq = lib.NewWaitQuit("mongodb outputer", -1) mo.mongosAddr, _ = config["mongos"] mo.db, _ = config["db"] mo.collection, _ = config["collection"] mo.session = initMongoDbSession(mo.mongosAddr) //暂时出错直接退出 if mo.session == nil { loglib.Error("init mongodb session failed") os.Exit(1) } upsert, _ := config["upsert"] if upsert == "true" { mo.isUpsert = true } else { mo.isUpsert = false } bulkSize, _ := config["bulk_size"] nBulk, err := strconv.Atoi(bulkSize) if err == nil { mo.bulkSize = nBulk } else { mo.bulkSize = 50 } savers, _ := config["savers"] nSavers, err := strconv.Atoi(savers) if err == nil { mo.savers = nSavers } else { mo.savers = 20 } //创建文件缓存目录 mo.file_mem_folder_name = "tempfile" if !lib.FileExists(mo.file_mem_folder_name) { os.MkdirAll(mo.file_mem_folder_name, 0775) } mo.transactionIdKey = "transaction_id" mo.fileList = lib.GlobalListInit() return mo }
func (f *fileOutputer) Start() { defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("file outputer panic:%v", err)) } f.ic.SaveStatus() f.closeWriters(f.writers) f.closeWriters(f.headerWriters) f.wq.AllDone() }() //使用range遍历,方便安全退出,只要发送方退出时关闭chan,这里就可以退出了 for b := range f.buffer { f.extract(&b) } }
func (t *TcpReceiver) Start() { tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress) lib.CheckError(err) listener, err := net.ListenTCP("tcp", tcpAddr) lib.CheckError(err) wg := &sync.WaitGroup{} wg.Add(1) go t.clearFootPrint(wg) //主routine信号处理 go lib.HandleQuitSignal(func() { //接收到信号关闭listenner,此时Accept会马上返回一个nil 的conn listener.Close() loglib.Info("close tcp receiver's listener.") }) defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err)) } loglib.Info("wait connections finish...") wg.Wait() loglib.Info("all connections have been processed. quit.") close(t.buffer) //关闭chan t.saveFootPrint() t.wq.AllDone() }() for { conn, err := listener.Accept() if conn == nil { break } lib.CheckError(err) wg.Add(1) go t.handleConnnection(conn, wg) } }
func createSingleConnection(address string) (conn *net.TCPConn, err error) { tcpAddr, err := net.ResolveTCPAddr("tcp4", address) lib.CheckError(err) if err != nil { return nil, err } conn, err = net.DialTCP("tcp", nil, tcpAddr) if err != nil { loglib.Error("get connection from " + address + " failed! Error:" + err.Error()) return nil, err } else { loglib.Info("get connection from " + address + " success! remote addr " + conn.RemoteAddr().String()) } lib.CheckError(err) return conn, nil }
func (this *MongoDbOutputer) extract(bp *bytes.Buffer) (r io.ReadCloser, packId string, date string, err error) { buf := make([]byte, 4) bp.Read(buf) l, _ := binary.Uvarint(buf) headerLen := int(l) //get pack header buf = make([]byte, headerLen) bp.Read(buf) header := tcp_pack.ParseHeader(buf) r, err = zlib.NewReader(bp) if err != nil { loglib.Error("zlib reader Error: " + err.Error()) } date = header["hour"][0:8] //用于按天分库 packId = fmt.Sprintf("%s_%s_%s", header["ip"], header["hour"], header["id"]) return }
func getIpRoleMap(dbConn *db.Mysql) map[string]string { sql := "select ip, port, role from " + registerTable res, err := dbConn.Query(sql) m := make(map[string]string) if err != nil { loglib.Error("read registered nodes failed! Error: " + err.Error()) } else { for _, row := range res.Rows { ip := row[0] port := row[1] role := row[2] if port != "" { ip = ip + ":" + port } m[ip] = role } loglib.Info("readed working nodes from database.") } return m }
//缓存写入mongodb失败的数据 //typeStr为bulk或upsert func (this *MongoDbOutputer) cacheData(data interface{}, typeStr string, date string, routineId int) { mp := bson.M{"type": typeStr, "date": date, "data": data} saveTry := 3 b, err := json.Marshal(mp) arr, ok := data.([]bson.M) cnt := 1 if ok { cnt = len(arr) } if err != nil { loglib.Error(fmt.Sprintf("cache data error when marshal, discard %d item(s), error:%v", cnt, err)) return } fname := this.createFileName(routineId) for i := 0; i < saveTry; i++ { err = ioutil.WriteFile(fname, b, 0666) if err == nil { this.fileList.PushBack(fname) loglib.Info(fmt.Sprintf("cache %d bson", cnt)) break } } }
//goroutine func (s *Sender) Start() { // conn := s.getConnection() //初始化fileCacheList once.Do(s.reloadFileCache) //收尾工作 defer func() { if err := recover(); err != nil { loglib.Error(fmt.Sprintf("sender %d panic:%v", s.id, err)) } s.saveBufferInChan() //s.saveMemCache() s.connection.close() s.wq.AllDone() }() go s.pickPacks() //var connLost = 0 var quit = false go lib.HandleQuitSignal(func() { quit = true s.connection.close() }) var sendInterval = time.Duration(2000) //间隔稍大,避免发送文件缓存时因无连接或其他错误进入死循环 var timeoutChan = time.After(sendInterval * time.Millisecond) for !quit { select { case b := <-s.memBuffer: //send b result := s.sendBuffer(b) if result == false { //改为直接放入文件缓存 s.writeToFile(b) } case <-timeoutChan: timeoutChan = time.After(sendInterval * time.Millisecond) // send from file e := fileList.Remove() if e != nil { // file list is not empty filename := e.Value.(string) // fmt.Println("sender ",s.id,": get file :",filename) data, err := ioutil.ReadFile(filename) if err != nil { // fmt.Println("sender ",s.id,":",err) if _, ok := err.(*os.PathError); !ok { fileList.PushBack(filename) } loglib.Error(fmt.Sprintf("sender%d read file cache %s error:%s", s.id, filename, err.Error())) } else { packId := tcp_pack.GetPackId(data) //debug info loglib.Info(fmt.Sprintf("sender%d read pack %s from file: %s, len: %d", s.id, packId, filename, len(data))) //debug info result := s.sendData2(data) if result == true { // s.fileCacheList.Remove(front) // log.Println("sender ",s.id,":removed file:",filename, "for pack", packId)//debug info err = os.Remove(filename) lib.CheckError(err) timeoutChan = time.After(time.Millisecond) //发送成功,不用再等待 } else { fileList.PushBack(filename) // fmt.Println("sender ",s.id,": pushback file :",filename) } } } } } }