func DoFetch(cf *config.Config) {
log.Println("=== FETCH BEGIN ===")
s := new(fetcher.Session)
s.Config = cf
for _, realm := range cf.RealmsList {
for _, locale := range cf.LocalesList {
file_url, file_ts := s.Fetch_FileURL(realm, locale)
log.Printf("FILE URL: %s", file_url)
log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
fname := util.Make_FName(realm, file_ts, true)
json_fname := cf.DownloadDirectory + fname
if !util.CheckFile(json_fname) {
log.Printf("downloading from %s ...", file_url)
data := s.Get(file_url)
log.Printf("... got %d octets", len(data))
zdata := util.Zip(data)
log.Printf("... zipped to %d octets (%d%%)",
len(zdata), len(zdata)*100/len(data))
util.Store(json_fname, zdata)
log.Printf("stored to %s .", json_fname)
} else {
log.Println("... already downloaded")
}
}
}
log.Println("=== FETCH END ===")
}
func ParseDir(cf *config.Config, realm string, safe bool) {
mask := cf.DownloadDirectory +
strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
log.Printf("scan by mask %s ...", mask)
fnames, err := filepath.Glob(mask)
if err != nil {
log.Fatalln("glob failed:", err)
}
log.Printf("... %d entries collected", len(fnames))
var goodfnames []string
for _, fname := range fnames {
// realm, ts, good := util.Parse_FName(fname)
_, _, good := util.Parse_FName(fname)
if good {
// log.Printf("fname %s -> %s, %v", fname, realm, ts)
goodfnames = append(goodfnames, fname)
} else {
// log.Printf("skip fname %s", fname)
}
}
sort.Sort(util.ByBasename(goodfnames))
prc := new(AuctionProcessor)
prc.Init(cf, realm)
prc.LoadState()
badfiles := make(map[string]string)
for _, fname := range fnames {
//log.Println(fname)
f_realm, f_time, ok := util.Parse_FName(fname)
if !ok {
log.Fatalf("not parsed correctly: %s", fname)
continue
}
if f_realm != realm {
log.Fatalf("not my realm (%s != %s)", f_realm, realm)
continue
}
if !prc.SnapshotNeeded(f_time) {
log.Printf("snapshot not needed: %s", util.TSStr(f_time))
continue
}
data, err := util.Load(fname)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s LOAD ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
ss, err := ParseSnapshot(data)
if err != nil {
//log.Fatalf("load error: %s", err)
log.Printf("%s PARSE ERROR: %s", fname, err)
badfiles[fname] = fmt.Sprint(err)
continue
}
prc.StartSnapshot(f_time)
for _, auc := range ss.Auctions {
prc.AddAuctionEntry(&auc)
}
prc.FinishSnapshot()
if safe {
prc.SaveState()
}
}
if !safe {
prc.SaveState()
}
if len(badfiles) == 0 {
log.Printf("all files loaded without errors")
} else {
log.Printf("%d files with errors", len(badfiles))
for fname, err := range badfiles {
log.Printf("%s: %s", fname, err)
}
}
}
func (prc *AuctionProcessor) FinishSnapshot() {
if !prc.Started {
log.Panic("FinishSnapshot outside snapshot session")
}
// log.Println("check for closed auctions")
num_open, num_closed := 0, 0
auc_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("auctions", prc.Realm, prc.SnapshotTime)
meta_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("metadata", prc.Realm, prc.SnapshotTime)
snap_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("snapshot", prc.Realm, prc.SnapshotTime)
prc.FileAuc = OpenOrCreateFile(auc_fname)
defer prc.FileAuc.Close()
prc.FileMeta = OpenOrCreateFile(meta_fname)
defer prc.FileMeta.Close()
SnapInfo := OpenOrCreateFile(snap_fname)
defer SnapInfo.Close()
for id, _ := range prc.State.WorkSet {
_, seen := prc.SeenSet[id]
if !seen {
num_closed++
prc.closeEntry(id)
} else {
num_open++
}
}
var rate int = 0
if num_closed > 0 {
rate = (prc.NumBought + prc.NumAuctioned) * 100 / num_closed
}
prc.TotalOpened += prc.NumCreated
prc.TotalClosed += num_closed
prc.TotalSuccess += prc.NumBought + prc.NumAuctioned
var total_rate int = 0
if prc.TotalClosed > 0 {
total_rate = prc.TotalSuccess * 100 / prc.TotalClosed
}
log.Printf("%s: \n"+
" entries: %d\n"+
" active: %d,\n"+
" created: %d,\n"+
" changed: %d [bids: %d, adj: %d, moves: %d]\n"+
" closed: %d [bought: %d, auctioned: %d, expired: %d, succes: %d%%]",
util.TSStr(prc.SnapshotTime),
len(prc.State.WorkSet), num_open,
prc.NumCreated, prc.NumModified,
prc.NumBids, prc.NumAdjusts, prc.NumMoves,
num_closed, prc.NumBought, prc.NumAuctioned, prc.NumExpired, rate)
log.Printf("total created %d, closed %d, success %d%%",
prc.TotalOpened, prc.TotalClosed, total_rate)
SnapInfo.WriteString(
fmt.Sprintf("%s: entries:%d active:%d created:%d "+
"changed:%d [bids:%d adj:%d moves:%d] "+
"closed:%d [bought:%d auctioned:%d expired:%d rate:%d%%]\n",
util.TSStr(prc.SnapshotTime),
len(prc.State.WorkSet), num_open,
prc.NumCreated, prc.NumModified,
prc.NumBids, prc.NumAdjusts, prc.NumMoves,
num_closed, prc.NumBought, prc.NumAuctioned, prc.NumExpired,
rate))
prc.State.LastTime = prc.SnapshotTime
//log.Printf("last time sets to %s", util.TSStr(prc.State.LastTime))
prc.Started = false
}
