Пример #1
0
func Reupload() {
	for {
		select {
		case redo := <-reuploadSet:
			time.Sleep(time.Second * 30)
			sec := redo.LogTime.Unix()
			sec = sec - sec%300 - 600
			logtime := time.Unix(sec, 0).Format("2006-01-02T15:04:00.000+0800")
			options.Log.Noticef("开始重传域名为[%v]时间点为[%v]的日志文件", redo.Host, logtime)

			if options.Config.Custom == "mangguo" {
				if err := options.PostFileInfo(redo.File, options.Config.MongguoPostFileUrl); err != nil {
					options.Log.Error(err)
					redo.RetryNum += 1
					reuploadSet <- redo
					continue
				} else {
					if err := options.CreateFinishFile(redo.LocalFinishFile); err != nil {
						options.Log.Errorf("重传时创建日志完成文件[%v]失败,错误原因:%v", redo.LocalFinishFile, err)
						redo.RetryNum += 1
						reuploadSet <- redo
						continue
					}
				}
				if err := options.CreateFinishFile(redo.LocalFinishFile); err != nil {
					options.Log.Errorf("重传时创建日志完成文件[%v]失败,错误原因:%v", redo.LocalFinishFile, err)
					redo.RetryNum += 1
					reuploadSet <- redo
					continue
				}
			}

			if redo.RetryNum < 4 {
				if err := options.Upload(
					redo.FtpLogFile,
					redo.LocalLogFile,
					redo.FtpFinishFile,
					redo.LocalFinishFile,
					redo.FtpDir,
					redo.Host,
					options.Config.Custom,
				); err != nil {
					options.Log.Errorf("上传域名为[%v]时间点为[%v]的日志文件失败,失败原因:%v", redo.Host, logtime, err)
					redo.RetryNum += 1
					reuploadSet <- redo
				} else {
					options.Log.Noticef("重传域名为[%v]时间点为[%v]的日志文件成功", redo.Host, logtime)
				}
			} else {
				options.Log.Warningf("域名为[%v]时间点为[%v]的日志文件重传次数已经超过[3]次,不会再重传", logtime)
			}
		}
	}
}
Пример #2
0
func SyncHdfsLogFile(currentTime time.Time) {
	dir := options.HdfsFileDir(options.Config.HdfsRoot, currentTime)
	finish := options.CheckFinish(dir)
	if !finish {
		cell := new(options.Cell)
		cell.LogTime = currentTime
		notFinish <- cell
	} else {
		names := options.GetFilePathSet(dir)
		if len(names) != 0 {
			for i := 0; i < len(names); i++ {
				for j := 0; j < len(options.Hosts); j++ {
					if strings.Contains(names[i], options.Hosts[j]) {
						local := options.TargetLocalPath(options.Hosts[j], options.Config.Custom, currentTime)
						if options.Config.Custom == "mangguo" {
							temp := strings.Replace(local, ":", "@", 1)
							if err := options.Download(names[i], temp); err != nil {
								continue
							}
							cmdargs := []string{temp, local}
							cmd := exec.Command("mv", cmdargs...)
							err := cmd.Run()
							if err != nil {
								options.Log.Errorf("重命名文件[%s]为[%s]失败, 错误原因 : %v", temp, local, err)
								continue
							}
						} else {
							if err := options.Download(names[i], local); err != nil {
								continue
							}
						}

						finfo, err := os.Stat(local)
						if err != nil {
							options.Log.Errorf("获取文件:%v 信息失败,错误原因:%v", local, err)
							continue
						}
						md5, err := options.GetFilemd5ByCmd(local)
						if err != nil {
							options.Log.Errorf("获取文件:%v 的md5值失败,错误原因:%v", local, err)
							continue
						}

						cell := new(options.Cell)
						cell.File = new(options.FileInfo)
						cell.File.BeginDataTime, cell.File.EndDataTime = options.CreateBeginAndEndTime(currentTime)
						cell.FtpLogFile, cell.LocalLogFile, cell.FtpFinishFile, cell.LocalFinishFile, cell.FtpDir = options.Factory(options.Hosts[j], options.Config.Custom, currentTime)
						cell.File.Domain = options.Hosts[j]
						cell.File.FileLength = finfo.Size()
						cell.File.FilePathName = cell.FtpLogFile
						cell.File.MD5 = md5
						cell.File.Provider = "yunfan"
						cell.Host = options.Hosts[j]
						cell.LogTime = currentTime
						if options.Config.Custom == "mangguo" {
							if err = options.PostFileInfo(cell.File, options.Config.MongguoPostFileUrl); err != nil {
								options.Log.Error(err)
								reuploadSet <- cell
								continue
							} else {
								if err := options.CreateFinishFile(cell.LocalFinishFile); err != nil {
									options.Log.Errorf("创建日志完成文件[%v]失败,错误原因:%v", cell.LocalFinishFile, err)
									reuploadSet <- cell
									continue
								}
							}
							if err := options.CreateFinishFile(cell.LocalFinishFile); err != nil {
								options.Log.Errorf("创建日志完成文件[%v]失败,错误原因:%v", cell.LocalFinishFile, err)
								reuploadSet <- cell
								continue
							}
						}
						uploadSet <- cell
					}
				}
			}
		} else {
			options.Log.Warningf("hdfs服务器目录[%v]中没有生成指定域名的日志文件", dir)
		}
	}
}