Exemple #1
0
func getFilePath() string {
	var d = lib.GetBinPath() + "/var"
	if !lib.FileExists(d) {
		os.MkdirAll(d, 0775)
	}
	return d + "/log_received.json"
}
Exemple #2
0
func MongoDbOutputerInit(buffer chan bytes.Buffer, config map[string]string) (mo MongoDbOutputer) {
    mo.buffer = buffer
    mo.wq = lib.NewWaitQuit("mongodb outputer", -1)
    mo.mongosAddr, _ = config["mongos"]
    mo.db         , _ = config["db"]
    mo.collection , _ = config["collection"]
    mo.session = initMongoDbSession(mo.mongosAddr)
    //暂时出错直接退出
    if mo.session == nil {
        loglib.Error("init mongodb session failed")
        os.Exit(1)
    }

    upsert, _ := config["upsert"]
    if upsert == "true" {
        mo.isUpsert = true
    }else{
        mo.isUpsert = false
    }

    bulkSize, _ := config["bulk_size"]
    nBulk, err := strconv.Atoi(bulkSize)
    if err == nil {
        mo.bulkSize = nBulk
    }else{
        mo.bulkSize = 50
    }

    savers, _ := config["savers"]
    nSavers, err := strconv.Atoi(savers)
    if err == nil {
        mo.savers = nSavers
    }else{
        mo.savers = 20
    }
    
    //创建文件缓存目录
    mo.file_mem_folder_name = "tempfile"
    if !lib.FileExists(mo.file_mem_folder_name) {
        os.MkdirAll(mo.file_mem_folder_name, 0775)
    }
    
    mo.transactionIdKey = "transaction_id"
    mo.fileList = lib.GlobalListInit()
    return mo
}
Exemple #3
0
func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
	fp := make(map[string]PackAppear)
	if lib.FileExists(fname) {
		vbytes, err := ioutil.ReadFile(fname)
		if err != nil {
			loglib.Error("read footprint file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &fp)
			if err != nil {
				loglib.Error("unmarshal footprint error:" + err.Error())
			} else {
				loglib.Info("load footprint success !")
			}
		}
	} else {
		loglib.Warning("footprint file " + fname + " not found!")
	}
	return fp
}
Exemple #4
0
func (this *IntegrityChecker) LoadStatus(filename string) map[string]map[string]map[string]map[string]int {
	m := make(map[string]map[string]map[string]map[string]int)
	m["hour_received"] = make(map[string]map[string]map[string]int)
	m["day_received"] = make(map[string]map[string]map[string]int)
	if lib.FileExists(filename) {
		vbytes, err := ioutil.ReadFile(filename)
		if err != nil {
			loglib.Error("read log received file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &m)
			if err != nil {
				loglib.Error("unmarshal log received error:" + err.Error())
			} else {
				loglib.Info("load log received success !")
			}
		}
	} else {
		loglib.Warning("log received file " + filename + " not found!")
	}
	return m
}
Exemple #5
0
func (this *Tailler) taillingCurrent(receiveChan chan map[string]string) {
	var n_lines = ""
	if this.lineNum >= 0 {
		n_lines = fmt.Sprintf("+%d", this.lineNum+1) //略过已经tail过的行
	} else {
		n_lines = "0" //从最后开始
	}

	var quit = false
	//收尾工作
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("tailler panic:%v", err))
		}

		//如果是quit,丢弃不完整的包
		if quit {
			this.lineNum -= this.lineNum % this.recvBufSize
		}
		saveLineRecord(this.recordPath, this.currFile, this.lineNum)

		this.wq.AllDone()
	}()

	//启动时读取行号,以后都从首行开始
	cmd := exec.Command("tail", "-F", "-n", n_lines, this.currFile)
	n_lines = "+1"
	stdout, err := cmd.StdoutPipe()

	if err != nil {
		loglib.Error("open pipe error")
	}

	//系统信号监听
	go lib.HandleQuitSignal(func() {
		quit = true
		if cmd.Process != nil {
			cmd.Process.Kill() //关闭tail命令,不然读取循环无法终止
		}
	})

	//日志切割检测
	go func() {
		nextHour := this.fileHour.Add(time.Hour)
		nextHourFile := this.getLogFileByTime(nextHour)
		timeToWait := 10 * time.Minute //到达下一小时后,等待日志文件的最长时间,10分钟
		for {
			if quit {
				break
			}
			if lib.FileExists(nextHourFile) || time.Now().Sub(nextHour) > timeToWait {
				currFile := this.currFile
				totalLines := this.GetTotalLines(currFile)
				loglib.Info(fmt.Sprintf("log rotated! previous file: %s, total lines: %d", currFile, totalLines))

				//在kill前进行文件切换,避免kill后新的tail启动时文件名还是旧的
				this.fileHour = nextHour
				this.currFile = nextHourFile
				nextHour = nextHour.Add(time.Hour)
				nextHourFile = this.getLogFileByTime(nextHour)

				//发现日志切割,等待1分钟
				i := 0
				done := false
				for {
					if this.lineNum >= totalLines {
						done = true
					}
					if done || i > 60 {
						if cmd.Process != nil {
							cmd.Process.Kill() //关闭tail命令,不然读取循环无法终止
						}
						if done {
							loglib.Info("finish tail " + currFile)
						} else {
							loglib.Info("tail " + currFile + " timeout")
						}
						break
					}
					i++
					time.Sleep(time.Second)
				}
			}
			time.Sleep(time.Second)
		}

	}()

outer:
	for {
		currFile := this.currFile //缓存当前tail的文件名
		hourStr := this.fileHour.Format(this.hourStrFmt)
		cmd.Start()
		loglib.Info("begin current log: " + currFile)
		rd := bufio.NewReader(stdout)
		for line, err := rd.ReadString('\n'); err == nil; line, err = rd.ReadString('\n') {
			//fmt.Print(line)
			if quit {
				break outer
			}
			this.lineNum++
			m := map[string]string{"hour": hourStr, "line": line}
			receiveChan <- m
			if this.lineNum%this.recvBufSize == 0 {
				saveLineRecord(this.recordPath, currFile, this.lineNum)
			}
		}
		if err := cmd.Wait(); err != nil {
			loglib.Info("wait sys tail error!" + err.Error())
		}
		loglib.Info(fmt.Sprintf("%s tailed %d lines", currFile, this.lineNum))
		if quit {
			break
		}
		// 完整tail一个文件
		m := map[string]string{"hour": hourStr, "line": changeStr}
		receiveChan <- m
		saveLineRecord(this.recordPath, currFile, this.lineNum)
		//begin a new file
		this.lineNum = 0
		cmd = exec.Command("tail", "-F", "-n", n_lines, this.currFile)
		stdout, err = cmd.StdoutPipe()

		if err != nil {
			loglib.Error("open pipe error")
			break
		}
	}
}