func MakeZip(zipname string, fnames []string) error {
log.Printf("zipping %d entrires to %s ...", len(fnames), zipname)
zipfile, err := os.Create(zipname)
if err != nil {
return err
}
defer zipfile.Close()
zipwriter := zip.NewWriter(zipfile)
defer zipwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
name := util.Make_FName(realm, ts, false)
f, err := zipwriter.Create(name)
if err != nil {
return err
}
log.Printf("zip %d bytes for file %s", len(data), name)
_, err = f.Write(data)
if err != nil {
return err
}
}
log.Printf("%s zipped without errors", zipname)
return nil
}
func MakeTarball(tarname string, fnames []string) error {
log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
tarfile, err := os.Create(tarname)
if err != nil {
return err
}
defer tarfile.Close()
var tarwriter *tar.Writer
if strings.HasSuffix(tarname, ".gz") {
zipper := gzip.NewWriter(tarfile)
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
/*
} else if strings.HasSuffix(tarname, ".xz") {
p := xz.WriterDefaults
p.DictCap = 1 << 24
zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
if err != nil {
return err
}
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
*/
} else {
tarwriter = tar.NewWriter(tarfile)
}
defer tarwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
hdr := new(tar.Header)
hdr.Name = util.Make_FName(realm, ts, false)
hdr.Size = int64(len(data))
hdr.ModTime = ts
hdr.Mode = 0644
err = tarwriter.WriteHeader(hdr)
if err != nil {
return err
}
log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
_, err = tarwriter.Write(data)
if err != nil {
return err
}
}
log.Printf("%s tarred without errors", tarname)
return nil
}
func Backup(srcdir, dstdir, timeformat, ext string) {
// Backup("/opt/wowauc/download", "/opt/wowauc/backup", "20060102", ".tar.gz")
fnames, err := filepath.Glob(srcdir + "/*.json.gz")
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
rmap := make(map[string][]string)
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
realm, ts, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
key := strings.Replace(realm, ":", "-", -1) + "-" + ts.Format(timeformat)
rmap[key] = append(rmap[key], fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
var keys []string
for key, _ := range rmap {
keys = append(keys, key)
}
sort.Sort(util.ByContent(keys))
for _, key := range keys {
fnames := rmap[key]
log.Printf("create tarball for %s with %d entrires ", key, len(fnames))
sort.Sort(util.ByBasename(fnames))
if ext == ".tar.gz" {
tarname := dstdir + "/" + key + ext
err := MakeTarball(tarname, fnames)
if err != nil {
log.Fatalf("MakeTarball(%s) failed: %s", tarname, err)
}
} else if ext == ".zip" {
zipname := dstdir + "/" + key + ext
err := MakeZip(zipname, fnames)
if err != nil {
log.Fatalf("MakeZip(%s) failed: %s", zipname, err)
}
}
}
return
}
func ParseDir(cf *config.Config, realm string, safe bool) {
mask := cf.DownloadDirectory +
strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
log.Printf("scan by mask %s ...", mask)
fnames, err := filepath.Glob(mask)
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
var goodfnames []string
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
_, _, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
goodfnames = append(goodfnames, fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
sort.Sort(util.ByBasename(goodfnames))
prc := new(AuctionProcessor)
prc.Init(cf, realm)
prc.LoadState()
badfiles := make(map[string]string)
for _, fname := range fnames {
//log.Println(fname)
f_realm, f_time, ok := util.Parse_FName(fname)
if !ok {
log.Fatalf("not parsed correctly: %s", fname)
continue
}
if f_realm != realm {
log.Fatalf("not my realm (%s != %s)", f_realm, realm)
continue
}
if !prc.SnapshotNeeded(f_time) {
log.Printf("snapshot not needed: %s", util.TSStr(f_time))
continue
}
data, err := util.Load(fname)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s LOAD ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
ss, err := ParseSnapshot(data)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s PARSE ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
prc.StartSnapshot(f_time)
for _, auc := range ss.Auctions {
prc.AddAuctionEntry(&auc)
}
prc.FinishSnapshot()
if safe {
prc.SaveState()
}
}
if !safe {
prc.SaveState()
}
if len(badfiles) == 0 {
log.Printf("all files loaded without errors")
} else {
log.Printf("%d files with errors", len(badfiles))
for fname, err := range badfiles {
log.Printf("%s: %s", fname, err)
}
}
}