func Backup(srcdir, dstdir, timeformat, ext string) {
// Backup("/opt/wowauc/download", "/opt/wowauc/backup", "20060102", ".tar.gz")
fnames, err := filepath.Glob(srcdir + "/*.json.gz")
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
rmap := make(map[string][]string)
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
realm, ts, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
key := strings.Replace(realm, ":", "-", -1) + "-" + ts.Format(timeformat)
rmap[key] = append(rmap[key], fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
var keys []string
for key, _ := range rmap {
keys = append(keys, key)
}
sort.Sort(util.ByContent(keys))
for _, key := range keys {
fnames := rmap[key]
log.Printf("create tarball for %s with %d entrires ", key, len(fnames))
sort.Sort(util.ByBasename(fnames))
if ext == ".tar.gz" {
tarname := dstdir + "/" + key + ext
err := MakeTarball(tarname, fnames)
if err != nil {
log.Fatalf("MakeTarball(%s) failed: %s", tarname, err)
}
} else if ext == ".zip" {
zipname := dstdir + "/" + key + ext
err := MakeZip(zipname, fnames)
if err != nil {
log.Fatalf("MakeZip(%s) failed: %s", zipname, err)
}
}
}
return
}
func ParseDir(cf *config.Config, realm string, safe bool) {
mask := cf.DownloadDirectory +
strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
log.Printf("scan by mask %s ...", mask)
fnames, err := filepath.Glob(mask)
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
var goodfnames []string
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
_, _, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
goodfnames = append(goodfnames, fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
sort.Sort(util.ByBasename(goodfnames))
prc := new(AuctionProcessor)
prc.Init(cf, realm)
prc.LoadState()
badfiles := make(map[string]string)
for _, fname := range fnames {
//log.Println(fname)
f_realm, f_time, ok := util.Parse_FName(fname)
if !ok {
log.Fatalf("not parsed correctly: %s", fname)
continue
}
if f_realm != realm {
log.Fatalf("not my realm (%s != %s)", f_realm, realm)
continue
}
if !prc.SnapshotNeeded(f_time) {
log.Printf("snapshot not needed: %s", util.TSStr(f_time))
continue
}
data, err := util.Load(fname)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s LOAD ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
ss, err := ParseSnapshot(data)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s PARSE ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
prc.StartSnapshot(f_time)
for _, auc := range ss.Auctions {
prc.AddAuctionEntry(&auc)
}
prc.FinishSnapshot()
if safe {
prc.SaveState()
}
}
if !safe {
prc.SaveState()
}
if len(badfiles) == 0 {
log.Printf("all files loaded without errors")
} else {
log.Printf("%d files with errors", len(badfiles))
for fname, err := range badfiles {
log.Printf("%s: %s", fname, err)
}
}
}