/*
* 行数为非负值表示已tail的行数
* 行数为负值则都将从文件末尾开始
* 自动保存的行数只可能是bufSize的倍数或者文件总行数
*/
func getLineRecord(path string) (line int, fname string) {
fin, err := os.Open(path)
if err != nil {
_, f, l, _ := runtime.Caller(0)
loglib.Error(fmt.Sprintf("%s:%d open line record `%s` error\n", f, l, path))
return -1, "" //从最后开始读
}
var txt string
var lineStr = ""
//只读第一行
scanner := bufio.NewScanner(fin)
for scanner.Scan() {
txt = strings.Trim(scanner.Text(), " ")
break
}
fin.Close()
parts := strings.Split(txt, " ")
if len(parts) == 2 {
fname = parts[0]
lineStr = parts[1]
} else {
lineStr = parts[0]
}
line, err = strconv.Atoi(lineStr)
if err != nil {
loglib.Error("convert line record error:" + err.Error())
line = -1
}
return line, fname
}
//保存footprint
func (t *TcpReceiver) saveFootPrint() {
vbytes, err := json.Marshal(t.footPrint)
if err != nil {
loglib.Error("marshal footprint error:" + err.Error())
return
}
err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664)
if err == nil {
loglib.Info("save footprint success !")
} else {
loglib.Error("save footprint error:" + err.Error())
}
}
func saveLineRecord(path string, fname string, lineNum int) {
fout, err := os.Create(path)
defer fout.Close()
if err != nil {
loglib.Error("save line record error: " + err.Error())
return
}
_, err = fmt.Fprintf(fout, "%s %d", fname, lineNum)
if err != nil {
loglib.Error("Write line record error" + err.Error())
return
}
loglib.Info("save line record success!")
}
func (this *IntegrityChecker) SaveStatus() {
m := make(map[string]map[string]map[string]map[string]int)
m["hour_received"] = this.hourReceived
m["day_received"] = this.dayReceived
vbytes, err := json.Marshal(m)
if err != nil {
loglib.Error("marshal log received error:" + err.Error())
return
}
err = ioutil.WriteFile(this.statusFile, vbytes, 0664)
if err == nil {
loglib.Info("save log received success !")
} else {
loglib.Error("save log received error:" + err.Error())
}
}
func (this *Tailler) Tailling(receiveChan chan map[string]string) {
if this.currFile == "" {
//兼容老格式，老格式无文件路径
this.currFile = this.getLogFileByTime(time.Now())
}
var err error
this.fileHour, err = this.getTimeFromLogName(this.currFile)
if err != nil {
loglib.Error("can't get time from current log file:" + this.currFile + "error:" + err.Error())
os.Exit(1)
}
isQuit := false
for time.Since(this.fileHour).Hours() >= 1 {
//说明重启时已经跟记录行号时不属于同一个小时了
isQuit = this.taillingPrevious(this.currFile, this.lineNum, this.fileHour.Format(this.hourStrFmt), receiveChan)
if isQuit {
break
}
//继续下一个小时
this.fileHour = this.fileHour.Add(time.Hour)
this.currFile = this.getLogFileByTime(this.fileHour)
this.lineNum = 0
}
if !isQuit {
//处理当前这个小时
this.taillingCurrent(receiveChan)
}
close(receiveChan)
this.wq.AllDone()
}
func (this *MongoDbOutputer) Start() {
wg := &sync.WaitGroup{}
defer func(){
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("mongodb outputer panic:%v", err))
}
if this.session != nil {
this.session.Close()
}
this.wq.AllDone()
}()
this.reloadFileCache()
go this.reConnMongoDb()
wg.Add(this.savers)
for i:=0; i<this.savers; i++ {
go this.runParse(i, wg)
}
nRetry := this.savers / 6 + 1
wg.Add(nRetry)
for i:=0; i<nRetry; i++ {
go this.retrySave(wg, i)
}
wg.Wait()
}
func (lr *LogReceiver) Run() {
l, err := net.Listen("tcp", fmt.Sprintf(":%d", lr.port))
if err != nil {
loglib.Error("[log receiver] " + err.Error())
return
}
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
loglib.Error("[log receiver] " + err.Error())
return
}
go lr.handleConnection(conn)
}
}
func (f *fileOutputer) extract(bp *bytes.Buffer) {
buf := make([]byte, 4)
bp.Read(buf)
l, _ := binary.Uvarint(buf)
headerLen := int(l)
//get pack header
buf = make([]byte, headerLen)
bp.Read(buf)
header := tcp_pack.ParseHeader(buf)
r, err := zlib.NewReader(bp)
if err != nil {
loglib.Error("zlib reader Error: " + err.Error())
} else {
lines, _ := strconv.Atoi(header["lines"])
done := false
if header["done"] == "1" {
done = true
}
f.ic.Add(header["ip"], header["hour"], header["id"], lines, done)
writerKey := header["ip"] + "_" + header["hour"]
fout := f.getWriter(f.writers, f.dataDir, writerKey)
//一头一尾写头信息，节省硬盘
buf = append(buf, '\n')
//fout.Write(buf)
nn, err := io.Copy(fout, r)
if err != nil {
loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
}
//fout.Write(buf)
//单独存一份header便于查数
fout = f.getWriter(f.headerWriters, f.headerDir, writerKey)
n, err := fout.Write(buf)
if err != nil {
loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
}
if done || time.Now().Unix() > f.checkTime.Unix() {
hourFinish, _ := f.ic.Check()
for ip, hours := range hourFinish {
for _, hour := range hours {
writerKey = ip + "_" + hour
}
}
f.closeWriters(f.writers)
f.closeWriters(f.headerWriters)
f.checkTime.Add(2 * time.Minute)
}
r.Close()
}
}
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
fp := make(map[string]PackAppear)
if lib.FileExists(fname) {
vbytes, err := ioutil.ReadFile(fname)
if err != nil {
loglib.Error("read footprint file error:" + err.Error())
} else {
err = json.Unmarshal(vbytes, &fp)
if err != nil {
loglib.Error("unmarshal footprint error:" + err.Error())
} else {
loglib.Info("load footprint success !")
}
}
} else {
loglib.Warning("footprint file " + fname + " not found!")
}
return fp
}
//init conn list from addrMap
func (sc *SingleConnection) initConnection() {
newConn, err := createSingleConnection(sc.currentAddr)
if err != nil {
loglib.Error("init err:" + err.Error())
} else {
sc.conn = newConn
}
lib.CheckError(err)
}
func initMongoDbSession(mongosAddr string) *mgo.Session {
session, err := mgo.Dial(mongosAddr)
if err != nil {
loglib.Error(fmt.Sprintf("init mongodb session error:%v", err))
return nil
}
session.SetMode(mgo.Monotonic, true) //设置read preference
session.SetSafe(&mgo.Safe{W:2}) //设置write concern
return session
}
func (e *etlOutputer) Start() {
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
}
e.wq.AllDone()
}()
spiderList, _ := e.config["spider_list"]
colsFile, _ := e.config["columns_file"]
hostsList, _ := e.config["hosts_white_list"]
ipBlackList, _ := e.config["ip_black_list"]
if colsFile != "" {
e.runEtl(spiderList, colsFile, hostsList, ipBlackList)
} else {
loglib.Error("[error] miss columns map file!")
}
}
func (this *MongoDbOutputer) bulkSaveBson(coll *mgo.Collection, docs ...interface{}) (err error) {
if coll != nil {
err = coll.Insert(docs...)
if err != nil {
tmp := make([]string, 0)
for _, doc := range docs {
m, _ := doc.(bson.M)
tid, _ := m[this.transactionIdKey].(string)
tmp = append(tmp, tid)
}
tids := strings.Join(tmp, ",")
loglib.Error(fmt.Sprintf("save %d bsons [%s] error:%v", len(docs), tids, err))
}
}else{
err = errors.New("bulk: collection is nil")
loglib.Error(fmt.Sprintf("save bsons error:%v", err))
}
return
}
//touch一个文件表明某一天接收完
func (this *IntegrityChecker) makeDayTag(ip string, day string) bool {
fname := fmt.Sprintf("%s_%s", ip, day)
filename := filepath.Join(this.dir, fname)
fout, err := os.Create(filename)
if err != nil {
loglib.Error("tag " + fname + " error: " + err.Error())
return false
} else {
fout.Close()
}
return true
}
func (this *Tailler) GetTotalLines(fname string) int {
cmd := exec.Command("/bin/sh", "-c", `wc -l `+fname+` | awk '{print $1}'`)
out, err := cmd.Output()
if err == nil {
n, err := strconv.Atoi(strings.Trim(string(out), "\n"))
if err != nil {
loglib.Error("trans total lines " + string(out) + " error: " + err.Error())
}
return n
}
return 0
}
func (this *IntegrityChecker) LoadStatus(filename string) map[string]map[string]map[string]map[string]int {
m := make(map[string]map[string]map[string]map[string]int)
m["hour_received"] = make(map[string]map[string]map[string]int)
m["day_received"] = make(map[string]map[string]map[string]int)
if lib.FileExists(filename) {
vbytes, err := ioutil.ReadFile(filename)
if err != nil {
loglib.Error("read log received file error:" + err.Error())
} else {
err = json.Unmarshal(vbytes, &m)
if err != nil {
loglib.Error("unmarshal log received error:" + err.Error())
} else {
loglib.Info("load log received success !")
}
}
} else {
loglib.Warning("log received file " + filename + " not found!")
}
return m
}
func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) {
m, _ := selector.(bson.M)
tid, _ := m[this.transactionIdKey].(string)
if coll != nil {
info, err = coll.Upsert(selector, doc)
if err != nil {
loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
}else{
if info.Updated > 0 {
loglib.Info(fmt.Sprintf("bson [%s] updated", tid))
}
}
}else{
info = &mgo.ChangeInfo{}
err = errors.New("upsert: collection is nil")
loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
}
return
}
func NewTailler(config map[string]string) *Tailler {
val, ok := config[logFileKey]
if !ok || val == "" {
loglib.Error("config need log_file!")
os.Exit(1)
}
logPath := val
val, ok = config[recordFileKey]
if !ok || val == "" {
config[recordFileKey] = getRecordPath()
}
lineNum, fname := getLineRecord(config[recordFileKey])
goFmt, nLT := extractTimeFmt(logPath)
if goFmt == "" {
loglib.Error("log path has no time format!")
os.Exit(1)
}
wq := lib.NewWaitQuit("tailler")
bufSize, _ := strconv.Atoi(config["recv_buffer_size"])
return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}
func (f *fileOutputer) getWriter(writers map[string]*os.File, parentDir string, key string) *os.File {
w, ok := writers[key]
if !ok || w == nil {
fname := filepath.Join(parentDir, key)
w1, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
writers[key] = w1
w = w1
if err != nil {
loglib.Error(fmt.Sprintf("file outputer create writer: %s error: %s", fname, err.Error()))
}
}
return w
}
func (e *etlOutputer) doEtl(fkeyChan chan string, logDataDir string, etlDir string, etlDoneDir string, etlFailDir string, spiderList string, colsFile string, hostsList string, ipBlackList string, wg *sync.WaitGroup) {
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("doEtl() panic:%v", err))
}
wg.Done()
}()
loglib.Info("etl routine start")
for fkey := range fkeyChan {
sv := etl.NewFileSaver(colsFile, etlDir, fkey)
d := etl.NewDispatcher(sv, 6, hostsList, ipBlackList)
g := etl.NewGlobalHao123(spiderList, 100, 200, 8, d)
go g.Start(false)
fname := filepath.Join(logDataDir, fkey)
loglib.Info("start etl for " + fname)
err := g.ParseFile(fname)
g.Wait()
// etl success
// mark success
if err == nil {
//采用循环，增加打tag的成功率
for i := 0; i < 5; i++ {
fd, err := os.Create(filepath.Join(etlDoneDir, fkey))
if err == nil {
fd.Close()
loglib.Info("finish etl for " + fname)
break
} else {
loglib.Warning("mark etl done for " + fname + " failed! error: " + err.Error())
}
}
} else {
//采用循环，增加打tag的成功率
for i := 0; i < 5; i++ {
fd, err := os.Create(filepath.Join(etlFailDir, fkey))
if err == nil {
fd.Close()
loglib.Info("failed etl for " + fname)
break
} else {
loglib.Warning("mark etl fail for " + fname + " failed! error: " + err.Error())
}
}
}
}
loglib.Info("etl routine finish")
}
func check(addr string, ch chan CheckResult) {
conn, err := net.Dial("tcp", addr)
if err != nil {
msg := "[heart beat] connect to " + addr + " error"
loglib.Error(msg)
ch <- CheckResult{addr, true, msg}
} else {
conn.SetReadDeadline(time.Now().Add(10 * time.Second)) //读超时
var buf = make([]byte, 2)
n, err := conn.Read(buf)
if err != nil {
msg := "[heart beat] read data from " + addr + " error"
loglib.Error(msg)
ch <- CheckResult{addr, true, msg}
}
if string(buf[:n]) == "!" {
ch <- CheckResult{addr, false, ""}
} else {
ch <- CheckResult{addr, true, "unknow return code"}
}
conn.Close()
}
}
func New(configFile string) *Monitor {
//new Monitor
monitor := new(Monitor)
monitor.configFile = configFile
config := lib.ReadConfig(configFile)
// heart beat checker
cfg, ok := config["heart_beat"]
if !ok {
loglib.Error("miss heart beat config")
return nil
}
checkInterval, _ := strconv.Atoi(cfg["check_interval"])
delete(cfg, "check_interval")
mutex := &sync.RWMutex{}
monitor.mutex = mutex
monitor.checkInterval = checkInterval
hbChecker := heart_beat.NewHeartBeatChecker()
monitor.hbChecker = hbChecker
//log receiver
cfg, ok = config["receiver"]
if !ok {
loglib.Error("miss receiver config!")
return nil
}
mysql := db.NewMysql(cfg["db_host"], cfg["db_port"], cfg["db_uname"], cfg["db_passwd"], cfg["db_db"], cfg["db_charset"])
monitor.dbConn = mysql
monitor.ipRoleMap = getIpRoleMap(mysql)
recvPort, _ := strconv.Atoi(cfg["recv_port"])
receiver := NewLogReceiver(recvPort, mysql, monitor.ipRoleMap, monitor.mutex)
monitor.receiver = receiver
return monitor
}
func (this *HeartBeat) Run() {
i := 0
//尝试注册3次
for i < 3 && !this.registerSelf(true) {
i++
time.Sleep(10 * time.Second)
}
l, err := net.Listen("tcp", ":"+this.port)
if err != nil {
loglib.Error("heart beat " + err.Error())
return
}
defer l.Close()
go lib.HandleQuitSignal(func() {
this.registerSelf(false)
l.Close()
loglib.Info("close heart beat listener.")
})
//heart beat 不是太重要，所以退出处理比较简单
for {
conn, err := l.Accept()
if conn == nil {
//listener关闭
break
}
if err != nil {
loglib.Error("heart beat " + err.Error())
break
}
go this.handleConnection(conn)
}
this.wq.AllDone()
}
//向monitor注册或取消注册，reg为true表示注册，否则是取消注册，返回true表示成功
func (this *HeartBeat) registerSelf(reg bool) bool {
conn, err := lib.GetConnection(this.monitorAddr)
if err != nil {
return false
}
defer conn.Close()
req := "register"
if !reg {
req = "unregister"
}
m := map[string]string{"req": req, "ip": lib.GetIp(), "port": this.port, "hostname": lib.GetHostname(), "role": this.role}
msg, err := json.Marshal(m)
if err != nil {
loglib.Error("marshal " + req + " info error " + err.Error())
return false
}
_, err = conn.Write(tcp_pack.Pack(msg))
if err != nil {
loglib.Error("send " + req + " info failed" + err.Error())
return false
} else {
plen, ret := tcp_pack.UnPack(conn)
if plen > 0 {
err = json.Unmarshal(ret, &m)
r, ok := m["err"]
if err == nil && ok && r == "0" {
loglib.Info(req + " to monitor success!")
return true
}
loglib.Error(req + " heart beat failed!")
}
}
return false
}
//从日志文件名获取时间，不依赖系统时间
func (this *Tailler) getTimeFromLogName(name string) (time.Time, error) {
size := len(name)
timePart := ""
if this.nLT[0] < size && this.nLT[1] < size {
timePart = name[this.nLT[0] : size-this.nLT[1]]
}
layout := this.goFmt
loc, _ := time.LoadLocation("Local")
t, err := time.ParseInLocation(layout, timePart, loc)
if err != nil {
loglib.Error("parse " + timePart + " against " + layout + " error:" + err.Error())
}
return t, err
}
func NewIntegrityChecker(dir string) *IntegrityChecker {
err := os.MkdirAll(dir, 0755)
if err != nil {
loglib.Error("make integrity dir error:" + err.Error())
return nil
}
statusFile := getFilePath()
ic := &IntegrityChecker{}
ic.dir = dir
ic.statusFile = statusFile
status := ic.LoadStatus(ic.statusFile)
ic.hourReceived = status["hour_received"]
ic.dayReceived = status["day_received"]
return ic
}
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
mo.buffer = buffer
mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
mo.mongosAddr, _ = config["mongos"]
mo.db , _ = config["db"]
mo.collection , _ = config["collection"]
mo.session = initMongoDbSession(mo.mongosAddr)
//暂时出错直接退出
if mo.session == nil {
loglib.Error("init mongodb session failed")
os.Exit(1)
}
upsert, _ := config["upsert"]
if upsert == "true" {
mo.isUpsert = true
}else{
mo.isUpsert = false
}
bulkSize, _ := config["bulk_size"]
nBulk, err := strconv.Atoi(bulkSize)
if err == nil {
mo.bulkSize = nBulk
}else{
mo.bulkSize = 50
}
savers, _ := config["savers"]
nSavers, err := strconv.Atoi(savers)
if err == nil {
mo.savers = nSavers
}else{
mo.savers = 20
}
//创建文件缓存目录
mo.file_mem_folder_name = "tempfile"
if !lib.FileExists(mo.file_mem_folder_name) {
os.MkdirAll(mo.file_mem_folder_name, 0775)
}
mo.transactionIdKey = "transaction_id"
mo.fileList = lib.GlobalListInit()
return mo
}
func (f *fileOutputer) Start() {
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("file outputer panic:%v", err))
}
f.ic.SaveStatus()
f.closeWriters(f.writers)
f.closeWriters(f.headerWriters)
f.wq.AllDone()
}()
//使用range遍历，方便安全退出，只要发送方退出时关闭chan，这里就可以退出了
for b := range f.buffer {
f.extract(&b)
}
}
func (t *TcpReceiver) Start() {
tcpAddr, err := net.ResolveTCPAddr("tcp4", t.receiveFromAddress)
lib.CheckError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
lib.CheckError(err)
wg := &sync.WaitGroup{}
wg.Add(1)
go t.clearFootPrint(wg)
//主routine信号处理
go lib.HandleQuitSignal(func() {
//接收到信号关闭listenner，此时Accept会马上返回一个nil 的conn
listener.Close()
loglib.Info("close tcp receiver's listener.")
})
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("tcp receiver panic:%v", err))
}
loglib.Info("wait connections finish...")
wg.Wait()
loglib.Info("all connections have been processed. quit.")
close(t.buffer) //关闭chan
t.saveFootPrint()
t.wq.AllDone()
}()
for {
conn, err := listener.Accept()
if conn == nil {
break
}
lib.CheckError(err)
wg.Add(1)
go t.handleConnnection(conn, wg)
}
}
func createSingleConnection(address string) (conn *net.TCPConn, err error) {
tcpAddr, err := net.ResolveTCPAddr("tcp4", address)
lib.CheckError(err)
if err != nil {
return nil, err
}
conn, err = net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
loglib.Error("get connection from " + address + " failed! Error:" + err.Error())
return nil, err
} else {
loglib.Info("get connection from " + address + " success! remote addr " + conn.RemoteAddr().String())
}
lib.CheckError(err)
return conn, nil
}