示例#1
0
func Backup(srcdir, dstdir, timeformat, ext string) {
	// Backup("/opt/wowauc/download", "/opt/wowauc/backup", "20060102", ".tar.gz")
	fnames, err := filepath.Glob(srcdir + "/*.json.gz")
	if err != nil {
		log.Fatalln("glob failed:", err)
	}
	log.Printf("... %d entries collected", len(fnames))

	rmap := make(map[string][]string)

	for _, fname := range fnames {
		// realm, ts, good := util.Parse_FName(fname)
		realm, ts, good := util.Parse_FName(fname)
		if good {
			// log.Printf("fname %s -> %s, %v", fname, realm, ts)
			key := strings.Replace(realm, ":", "-", -1) + "-" + ts.Format(timeformat)
			rmap[key] = append(rmap[key], fname)
		} else {
			// log.Printf("skip fname %s", fname)
		}
	}

	var keys []string
	for key, _ := range rmap {
		keys = append(keys, key)
	}
	sort.Sort(util.ByContent(keys))

	for _, key := range keys {
		fnames := rmap[key]
		log.Printf("create tarball for %s with %d entrires ", key, len(fnames))
		sort.Sort(util.ByBasename(fnames))
		if ext == ".tar.gz" {
			tarname := dstdir + "/" + key + ext
			err := MakeTarball(tarname, fnames)
			if err != nil {
				log.Fatalf("MakeTarball(%s) failed: %s", tarname, err)
			}
		} else if ext == ".zip" {
			zipname := dstdir + "/" + key + ext
			err := MakeZip(zipname, fnames)
			if err != nil {
				log.Fatalf("MakeZip(%s) failed: %s", zipname, err)
			}
		}
	}
	return
}
示例#2
0
func ParseDir(cf *config.Config, realm string, safe bool) {
	mask := cf.DownloadDirectory +
		strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
	log.Printf("scan by mask %s ...", mask)
	fnames, err := filepath.Glob(mask)
	if err != nil {
		log.Fatalln("glob failed:", err)
	}
	log.Printf("... %d entries collected", len(fnames))

	var goodfnames []string

	for _, fname := range fnames {
		// realm, ts, good := util.Parse_FName(fname)
		_, _, good := util.Parse_FName(fname)
		if good {
			// log.Printf("fname %s -> %s, %v", fname, realm, ts)
			goodfnames = append(goodfnames, fname)
		} else {
			// log.Printf("skip fname %s", fname)
		}
	}
	sort.Sort(util.ByBasename(goodfnames))
	prc := new(AuctionProcessor)
	prc.Init(cf, realm)
	prc.LoadState()
	badfiles := make(map[string]string)

	for _, fname := range fnames {
		//log.Println(fname)
		f_realm, f_time, ok := util.Parse_FName(fname)
		if !ok {
			log.Fatalf("not parsed correctly: %s", fname)
			continue
		}
		if f_realm != realm {
			log.Fatalf("not my realm (%s != %s)", f_realm, realm)
			continue
		}
		if !prc.SnapshotNeeded(f_time) {
			log.Printf("snapshot not needed: %s", util.TSStr(f_time))
			continue
		}
		data, err := util.Load(fname)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s LOAD ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}
		ss, err := ParseSnapshot(data)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s PARSE ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}

		prc.StartSnapshot(f_time)
		for _, auc := range ss.Auctions {
			prc.AddAuctionEntry(&auc)
		}
		prc.FinishSnapshot()
		if safe {
			prc.SaveState()
		}
	}
	if !safe {
		prc.SaveState()
	}
	if len(badfiles) == 0 {
		log.Printf("all files loaded without errors")
	} else {
		log.Printf("%d files with errors", len(badfiles))
		for fname, err := range badfiles {
			log.Printf("%s: %s", fname, err)
		}
	}
}