func MakeZip(zipname string, fnames []string) error {
log.Printf("zipping %d entrires to %s ...", len(fnames), zipname)
zipfile, err := os.Create(zipname)
if err != nil {
return err
}
defer zipfile.Close()
zipwriter := zip.NewWriter(zipfile)
defer zipwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
name := util.Make_FName(realm, ts, false)
f, err := zipwriter.Create(name)
if err != nil {
return err
}
log.Printf("zip %d bytes for file %s", len(data), name)
_, err = f.Write(data)
if err != nil {
return err
}
}
log.Printf("%s zipped without errors", zipname)
return nil
}
func MakeTarball(tarname string, fnames []string) error {
log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
tarfile, err := os.Create(tarname)
if err != nil {
return err
}
defer tarfile.Close()
var tarwriter *tar.Writer
if strings.HasSuffix(tarname, ".gz") {
zipper := gzip.NewWriter(tarfile)
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
/*
} else if strings.HasSuffix(tarname, ".xz") {
p := xz.WriterDefaults
p.DictCap = 1 << 24
zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
if err != nil {
return err
}
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
*/
} else {
tarwriter = tar.NewWriter(tarfile)
}
defer tarwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
hdr := new(tar.Header)
hdr.Name = util.Make_FName(realm, ts, false)
hdr.Size = int64(len(data))
hdr.ModTime = ts
hdr.Mode = 0644
err = tarwriter.WriteHeader(hdr)
if err != nil {
return err
}
log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
_, err = tarwriter.Write(data)
if err != nil {
return err
}
}
log.Printf("%s tarred without errors", tarname)
return nil
}
func (prc *AuctionProcessor) LoadState() {
if prc.Started {
log.Panic("LoadState inside snapshot session")
}
if util.CheckFile(prc.StateFName) {
log.Printf("AuctionProcessor loading state from %s ...", prc.StateFName)
data, _ := util.Load(prc.StateFName)
if err := json.Unmarshal(data, &prc.State); err != nil {
log.Panicf("... failed: %s", prc.StateFName, err)
}
log.Printf("... loaded with %d list enties", len(prc.State.WorkList))
prc.State.WorkSet = make(WorkSetType)
for _, e := range prc.State.WorkList {
prc.State.WorkSet[e.Entry.Auc] = e
}
} else {
log.Printf("AuctionProcessor has no state named %s ...", prc.StateFName)
}
}
func ParseDir(cf *config.Config, realm string, safe bool) {
mask := cf.DownloadDirectory +
strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
log.Printf("scan by mask %s ...", mask)
fnames, err := filepath.Glob(mask)
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
var goodfnames []string
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
_, _, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
goodfnames = append(goodfnames, fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
sort.Sort(util.ByBasename(goodfnames))
prc := new(AuctionProcessor)
prc.Init(cf, realm)
prc.LoadState()
badfiles := make(map[string]string)
for _, fname := range fnames {
//log.Println(fname)
f_realm, f_time, ok := util.Parse_FName(fname)
if !ok {
log.Fatalf("not parsed correctly: %s", fname)
continue
}
if f_realm != realm {
log.Fatalf("not my realm (%s != %s)", f_realm, realm)
continue
}
if !prc.SnapshotNeeded(f_time) {
log.Printf("snapshot not needed: %s", util.TSStr(f_time))
continue
}
data, err := util.Load(fname)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s LOAD ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
ss, err := ParseSnapshot(data)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s PARSE ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
prc.StartSnapshot(f_time)
for _, auc := range ss.Auctions {
prc.AddAuctionEntry(&auc)
}
prc.FinishSnapshot()
if safe {
prc.SaveState()
}
}
if !safe {
prc.SaveState()
}
if len(badfiles) == 0 {
log.Printf("all files loaded without errors")
} else {
log.Printf("%d files with errors", len(badfiles))
for fname, err := range badfiles {
log.Printf("%s: %s", fname, err)
}
}
}