func Reupload() {
for {
select {
case redo := <-reuploadSet:
time.Sleep(time.Second * 30)
sec := redo.LogTime.Unix()
sec = sec - sec%300 - 600
logtime := time.Unix(sec, 0).Format("2006-01-02T15:04:00.000+0800")
options.Log.Noticef("开始重传域名为[%v]时间点为[%v]的日志文件", redo.Host, logtime)
if options.Config.Custom == "mangguo" {
if err := options.PostFileInfo(redo.File, options.Config.MongguoPostFileUrl); err != nil {
options.Log.Error(err)
redo.RetryNum += 1
reuploadSet <- redo
continue
} else {
if err := options.CreateFinishFile(redo.LocalFinishFile); err != nil {
options.Log.Errorf("重传时创建日志完成文件[%v]失败，错误原因：%v", redo.LocalFinishFile, err)
redo.RetryNum += 1
reuploadSet <- redo
continue
}
}
if err := options.CreateFinishFile(redo.LocalFinishFile); err != nil {
options.Log.Errorf("重传时创建日志完成文件[%v]失败，错误原因：%v", redo.LocalFinishFile, err)
redo.RetryNum += 1
reuploadSet <- redo
continue
}
}
if redo.RetryNum < 4 {
if err := options.Upload(
redo.FtpLogFile,
redo.LocalLogFile,
redo.FtpFinishFile,
redo.LocalFinishFile,
redo.FtpDir,
redo.Host,
options.Config.Custom,
); err != nil {
options.Log.Errorf("上传域名为[%v]时间点为[%v]的日志文件失败，失败原因：%v", redo.Host, logtime, err)
redo.RetryNum += 1
reuploadSet <- redo
} else {
options.Log.Noticef("重传域名为[%v]时间点为[%v]的日志文件成功", redo.Host, logtime)
}
} else {
options.Log.Warningf("域名为[%v]时间点为[%v]的日志文件重传次数已经超过[3]次，不会再重传", logtime)
}
}
}
}
func SyncHdfsLogFile(currentTime time.Time) {
dir := options.HdfsFileDir(options.Config.HdfsRoot, currentTime)
finish := options.CheckFinish(dir)
if !finish {
cell := new(options.Cell)
cell.LogTime = currentTime
notFinish <- cell
} else {
names := options.GetFilePathSet(dir)
if len(names) != 0 {
for i := 0; i < len(names); i++ {
for j := 0; j < len(options.Hosts); j++ {
if strings.Contains(names[i], options.Hosts[j]) {
local := options.TargetLocalPath(options.Hosts[j], options.Config.Custom, currentTime)
if options.Config.Custom == "mangguo" {
temp := strings.Replace(local, ":", "@", 1)
if err := options.Download(names[i], temp); err != nil {
continue
}
cmdargs := []string{temp, local}
cmd := exec.Command("mv", cmdargs...)
err := cmd.Run()
if err != nil {
options.Log.Errorf("重命名文件[%s]为[%s]失败, 错误原因 : %v", temp, local, err)
continue
}
} else {
if err := options.Download(names[i], local); err != nil {
continue
}
}
finfo, err := os.Stat(local)
if err != nil {
options.Log.Errorf("获取文件：%v 信息失败，错误原因：%v", local, err)
continue
}
md5, err := options.GetFilemd5ByCmd(local)
if err != nil {
options.Log.Errorf("获取文件：%v 的md5值失败，错误原因：%v", local, err)
continue
}
cell := new(options.Cell)
cell.File = new(options.FileInfo)
cell.File.BeginDataTime, cell.File.EndDataTime = options.CreateBeginAndEndTime(currentTime)
cell.FtpLogFile, cell.LocalLogFile, cell.FtpFinishFile, cell.LocalFinishFile, cell.FtpDir = options.Factory(options.Hosts[j], options.Config.Custom, currentTime)
cell.File.Domain = options.Hosts[j]
cell.File.FileLength = finfo.Size()
cell.File.FilePathName = cell.FtpLogFile
cell.File.MD5 = md5
cell.File.Provider = "yunfan"
cell.Host = options.Hosts[j]
cell.LogTime = currentTime
if options.Config.Custom == "mangguo" {
if err = options.PostFileInfo(cell.File, options.Config.MongguoPostFileUrl); err != nil {
options.Log.Error(err)
reuploadSet <- cell
continue
} else {
if err := options.CreateFinishFile(cell.LocalFinishFile); err != nil {
options.Log.Errorf("创建日志完成文件[%v]失败，错误原因：%v", cell.LocalFinishFile, err)
reuploadSet <- cell
continue
}
}
if err := options.CreateFinishFile(cell.LocalFinishFile); err != nil {
options.Log.Errorf("创建日志完成文件[%v]失败，错误原因：%v", cell.LocalFinishFile, err)
reuploadSet <- cell
continue
}
}
uploadSet <- cell
}
}
}
} else {
options.Log.Warningf("hdfs服务器目录[%v]中没有生成指定域名的日志文件", dir)
}
}
}