func MakeZip(zipname string, fnames []string) error { log.Printf("zipping %d entrires to %s ...", len(fnames), zipname) zipfile, err := os.Create(zipname) if err != nil { return err } defer zipfile.Close() zipwriter := zip.NewWriter(zipfile) defer zipwriter.Close() for _, fname := range fnames { realm, ts, good := util.Parse_FName(fname) if !good { log.Printf("warning: skip ill-named file '%s'", fname) continue // skip } data, err := util.Load(fname) if err != nil { return err } name := util.Make_FName(realm, ts, false) f, err := zipwriter.Create(name) if err != nil { return err } log.Printf("zip %d bytes for file %s", len(data), name) _, err = f.Write(data) if err != nil { return err } } log.Printf("%s zipped without errors", zipname) return nil }
func MakeTarball(tarname string, fnames []string) error { log.Printf("tarring %d entrires to %s ...", len(fnames), tarname) tarfile, err := os.Create(tarname) if err != nil { return err } defer tarfile.Close() var tarwriter *tar.Writer if strings.HasSuffix(tarname, ".gz") { zipper := gzip.NewWriter(tarfile) defer zipper.Close() tarwriter = tar.NewWriter(zipper) /* } else if strings.HasSuffix(tarname, ".xz") { p := xz.WriterDefaults p.DictCap = 1 << 24 zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile) if err != nil { return err } defer zipper.Close() tarwriter = tar.NewWriter(zipper) */ } else { tarwriter = tar.NewWriter(tarfile) } defer tarwriter.Close() for _, fname := range fnames { realm, ts, good := util.Parse_FName(fname) if !good { log.Printf("warning: skip ill-named file '%s'", fname) continue // skip } data, err := util.Load(fname) if err != nil { return err } hdr := new(tar.Header) hdr.Name = util.Make_FName(realm, ts, false) hdr.Size = int64(len(data)) hdr.ModTime = ts hdr.Mode = 0644 err = tarwriter.WriteHeader(hdr) if err != nil { return err } log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name) _, err = tarwriter.Write(data) if err != nil { return err } } log.Printf("%s tarred without errors", tarname) return nil }
func Backup(srcdir, dstdir, timeformat, ext string) { // Backup("/opt/wowauc/download", "/opt/wowauc/backup", "20060102", ".tar.gz") fnames, err := filepath.Glob(srcdir + "/*.json.gz") if err != nil { log.Fatalln("glob failed:", err) } log.Printf("... %d entries collected", len(fnames)) rmap := make(map[string][]string) for _, fname := range fnames { // realm, ts, good := util.Parse_FName(fname) realm, ts, good := util.Parse_FName(fname) if good { // log.Printf("fname %s -> %s, %v", fname, realm, ts) key := strings.Replace(realm, ":", "-", -1) + "-" + ts.Format(timeformat) rmap[key] = append(rmap[key], fname) } else { // log.Printf("skip fname %s", fname) } } var keys []string for key, _ := range rmap { keys = append(keys, key) } sort.Sort(util.ByContent(keys)) for _, key := range keys { fnames := rmap[key] log.Printf("create tarball for %s with %d entrires ", key, len(fnames)) sort.Sort(util.ByBasename(fnames)) if ext == ".tar.gz" { tarname := dstdir + "/" + key + ext err := MakeTarball(tarname, fnames) if err != nil { log.Fatalf("MakeTarball(%s) failed: %s", tarname, err) } } else if ext == ".zip" { zipname := dstdir + "/" + key + ext err := MakeZip(zipname, fnames) if err != nil { log.Fatalf("MakeZip(%s) failed: %s", zipname, err) } } } return }
func ParseDir(cf *config.Config, realm string, safe bool) { mask := cf.DownloadDirectory + strings.Replace(realm, ":", "-", -1) + "-*.json.gz" log.Printf("scan by mask %s ...", mask) fnames, err := filepath.Glob(mask) if err != nil { log.Fatalln("glob failed:", err) } log.Printf("... %d entries collected", len(fnames)) var goodfnames []string for _, fname := range fnames { // realm, ts, good := util.Parse_FName(fname) _, _, good := util.Parse_FName(fname) if good { // log.Printf("fname %s -> %s, %v", fname, realm, ts) goodfnames = append(goodfnames, fname) } else { // log.Printf("skip fname %s", fname) } } sort.Sort(util.ByBasename(goodfnames)) prc := new(AuctionProcessor) prc.Init(cf, realm) prc.LoadState() badfiles := make(map[string]string) for _, fname := range fnames { //log.Println(fname) f_realm, f_time, ok := util.Parse_FName(fname) if !ok { log.Fatalf("not parsed correctly: %s", fname) continue } if f_realm != realm { log.Fatalf("not my realm (%s != %s)", f_realm, realm) continue } if !prc.SnapshotNeeded(f_time) { log.Printf("snapshot not needed: %s", util.TSStr(f_time)) continue } data, err := util.Load(fname) if err != nil { //log.Fatalf("load error: %s", err) log.Printf("%s LOAD ERROR: %s", fname, err) badfiles[fname] = fmt.Sprint(err) continue } ss, err := ParseSnapshot(data) if err != nil { //log.Fatalf("load error: %s", err) log.Printf("%s PARSE ERROR: %s", fname, err) badfiles[fname] = fmt.Sprint(err) continue } prc.StartSnapshot(f_time) for _, auc := range ss.Auctions { prc.AddAuctionEntry(&auc) } prc.FinishSnapshot() if safe { prc.SaveState() } } if !safe { prc.SaveState() } if len(badfiles) == 0 { log.Printf("all files loaded without errors") } else { log.Printf("%d files with errors", len(badfiles)) for fname, err := range badfiles { log.Printf("%s: %s", fname, err) } } }