func DoFetch(cf *config.Config) {
log.Println("=== FETCH BEGIN ===")
s := new(fetcher.Session)
s.Config = cf
for _, realm := range cf.RealmsList {
for _, locale := range cf.LocalesList {
file_url, file_ts := s.Fetch_FileURL(realm, locale)
log.Printf("FILE URL: %s", file_url)
log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
fname := util.Make_FName(realm, file_ts, true)
json_fname := cf.DownloadDirectory + fname
if !util.CheckFile(json_fname) {
log.Printf("downloading from %s ...", file_url)
data := s.Get(file_url)
log.Printf("... got %d octets", len(data))
zdata := util.Zip(data)
log.Printf("... zipped to %d octets (%d%%)",
len(zdata), len(zdata)*100/len(data))
util.Store(json_fname, zdata)
log.Printf("stored to %s .", json_fname)
} else {
log.Println("... already downloaded")
}
}
}
log.Println("=== FETCH END ===")
}
func (prc *AuctionProcessor) SaveState() {
if prc.Started {
log.Panic("SaveState inside snapshot session")
}
log.Printf("AuctionProcessor storing state to %s ...", prc.StateFName)
log.Printf("... prepare list with %d enties", len(prc.State.WorkSet))
prc.State.WorkList = WorkListType{}
for _, e := range prc.State.WorkSet {
prc.State.WorkList = append(prc.State.WorkList, e)
}
data, err := json.Marshal(&prc.State)
if err != nil {
log.Fatalf("... failed: %s", err)
}
if strings.HasSuffix(prc.StateFName, ".gz") {
zdata := util.Zip(data)
log.Printf("store gzipped (%d%%) data to %s...",
len(zdata)*100/len(data), prc.StateFName)
util.Store(prc.StateFName, zdata)
} else {
log.Printf("store ungzipped data to %s...", prc.StateFName)
util.Store(prc.StateFName, data)
}
}