func getFilePath() string {
var d = lib.GetBinPath() + "/var"
if !lib.FileExists(d) {
os.MkdirAll(d, 0775)
}
return d + "/log_received.json"
}
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
mo.buffer = buffer
mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
mo.mongosAddr, _ = config["mongos"]
mo.db , _ = config["db"]
mo.collection , _ = config["collection"]
mo.session = initMongoDbSession(mo.mongosAddr)
//暂时出错直接退出
if mo.session == nil {
loglib.Error("init mongodb session failed")
os.Exit(1)
}
upsert, _ := config["upsert"]
if upsert == "true" {
mo.isUpsert = true
}else{
mo.isUpsert = false
}
bulkSize, _ := config["bulk_size"]
nBulk, err := strconv.Atoi(bulkSize)
if err == nil {
mo.bulkSize = nBulk
}else{
mo.bulkSize = 50
}
savers, _ := config["savers"]
nSavers, err := strconv.Atoi(savers)
if err == nil {
mo.savers = nSavers
}else{
mo.savers = 20
}
//创建文件缓存目录
mo.file_mem_folder_name = "tempfile"
if !lib.FileExists(mo.file_mem_folder_name) {
os.MkdirAll(mo.file_mem_folder_name, 0775)
}
mo.transactionIdKey = "transaction_id"
mo.fileList = lib.GlobalListInit()
return mo
}
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
fp := make(map[string]PackAppear)
if lib.FileExists(fname) {
vbytes, err := ioutil.ReadFile(fname)
if err != nil {
loglib.Error("read footprint file error:" + err.Error())
} else {
err = json.Unmarshal(vbytes, &fp)
if err != nil {
loglib.Error("unmarshal footprint error:" + err.Error())
} else {
loglib.Info("load footprint success !")
}
}
} else {
loglib.Warning("footprint file " + fname + " not found!")
}
return fp
}
func (this *IntegrityChecker) LoadStatus(filename string) map[string]map[string]map[string]map[string]int {
m := make(map[string]map[string]map[string]map[string]int)
m["hour_received"] = make(map[string]map[string]map[string]int)
m["day_received"] = make(map[string]map[string]map[string]int)
if lib.FileExists(filename) {
vbytes, err := ioutil.ReadFile(filename)
if err != nil {
loglib.Error("read log received file error:" + err.Error())
} else {
err = json.Unmarshal(vbytes, &m)
if err != nil {
loglib.Error("unmarshal log received error:" + err.Error())
} else {
loglib.Info("load log received success !")
}
}
} else {
loglib.Warning("log received file " + filename + " not found!")
}
return m
}
func (this *Tailler) taillingCurrent(receiveChan chan map[string]string) {
var n_lines = ""
if this.lineNum >= 0 {
n_lines = fmt.Sprintf("+%d", this.lineNum+1) //略过已经tail过的行
} else {
n_lines = "0" //从最后开始
}
var quit = false
//收尾工作
defer func() {
if err := recover(); err != nil {
loglib.Error(fmt.Sprintf("tailler panic:%v", err))
}
//如果是quit，丢弃不完整的包
if quit {
this.lineNum -= this.lineNum % this.recvBufSize
}
saveLineRecord(this.recordPath, this.currFile, this.lineNum)
this.wq.AllDone()
}()
//启动时读取行号，以后都从首行开始
cmd := exec.Command("tail", "-F", "-n", n_lines, this.currFile)
n_lines = "+1"
stdout, err := cmd.StdoutPipe()
if err != nil {
loglib.Error("open pipe error")
}
//系统信号监听
go lib.HandleQuitSignal(func() {
quit = true
if cmd.Process != nil {
cmd.Process.Kill() //关闭tail命令，不然读取循环无法终止
}
})
//日志切割检测
go func() {
nextHour := this.fileHour.Add(time.Hour)
nextHourFile := this.getLogFileByTime(nextHour)
timeToWait := 10 * time.Minute //到达下一小时后，等待日志文件的最长时间，10分钟
for {
if quit {
break
}
if lib.FileExists(nextHourFile) || time.Now().Sub(nextHour) > timeToWait {
currFile := this.currFile
totalLines := this.GetTotalLines(currFile)
loglib.Info(fmt.Sprintf("log rotated! previous file: %s, total lines: %d", currFile, totalLines))
//在kill前进行文件切换，避免kill后新的tail启动时文件名还是旧的
this.fileHour = nextHour
this.currFile = nextHourFile
nextHour = nextHour.Add(time.Hour)
nextHourFile = this.getLogFileByTime(nextHour)
//发现日志切割，等待1分钟
i := 0
done := false
for {
if this.lineNum >= totalLines {
done = true
}
if done || i > 60 {
if cmd.Process != nil {
cmd.Process.Kill() //关闭tail命令，不然读取循环无法终止
}
if done {
loglib.Info("finish tail " + currFile)
} else {
loglib.Info("tail " + currFile + " timeout")
}
break
}
i++
time.Sleep(time.Second)
}
}
time.Sleep(time.Second)
}
}()
outer:
for {
currFile := this.currFile //缓存当前tail的文件名
hourStr := this.fileHour.Format(this.hourStrFmt)
cmd.Start()
loglib.Info("begin current log: " + currFile)
rd := bufio.NewReader(stdout)
for line, err := rd.ReadString('\n'); err == nil; line, err = rd.ReadString('\n') {
//fmt.Print(line)
if quit {
break outer
}
this.lineNum++
m := map[string]string{"hour": hourStr, "line": line}
receiveChan <- m
if this.lineNum%this.recvBufSize == 0 {
saveLineRecord(this.recordPath, currFile, this.lineNum)
}
}
if err := cmd.Wait(); err != nil {
loglib.Info("wait sys tail error!" + err.Error())
}
loglib.Info(fmt.Sprintf("%s tailed %d lines", currFile, this.lineNum))
if quit {
break
}
// 完整tail一个文件
m := map[string]string{"hour": hourStr, "line": changeStr}
receiveChan <- m
saveLineRecord(this.recordPath, currFile, this.lineNum)
//begin a new file
this.lineNum = 0
cmd = exec.Command("tail", "-F", "-n", n_lines, this.currFile)
stdout, err = cmd.StdoutPipe()
if err != nil {
loglib.Error("open pipe error")
break
}
}
}