Ejemplo n.º 1
0
func DoFetch(cf *config.Config) {
	log.Println("=== FETCH BEGIN ===")
	s := new(fetcher.Session)
	s.Config = cf
	for _, realm := range cf.RealmsList {
		for _, locale := range cf.LocalesList {
			file_url, file_ts := s.Fetch_FileURL(realm, locale)
			log.Printf("FILE URL: %s", file_url)
			log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
			fname := util.Make_FName(realm, file_ts, true)
			json_fname := cf.DownloadDirectory + fname
			if !util.CheckFile(json_fname) {
				log.Printf("downloading from %s ...", file_url)
				data := s.Get(file_url)
				log.Printf("... got %d octets", len(data))
				zdata := util.Zip(data)
				log.Printf("... zipped to %d octets (%d%%)",
					len(zdata), len(zdata)*100/len(data))
				util.Store(json_fname, zdata)
				log.Printf("stored to %s .", json_fname)
			} else {
				log.Println("... already downloaded")
			}
		}
	}
	log.Println("=== FETCH END ===")
}
Ejemplo n.º 2
0
func (prc *AuctionProcessor) SaveState() {
	if prc.Started {
		log.Panic("SaveState inside snapshot session")
	}
	log.Printf("AuctionProcessor storing state to %s ...", prc.StateFName)
	log.Printf("... prepare list with %d enties", len(prc.State.WorkSet))
	prc.State.WorkList = WorkListType{}
	for _, e := range prc.State.WorkSet {
		prc.State.WorkList = append(prc.State.WorkList, e)
	}
	data, err := json.Marshal(&prc.State)
	if err != nil {
		log.Fatalf("... failed: %s", err)
	}
	if strings.HasSuffix(prc.StateFName, ".gz") {
		zdata := util.Zip(data)
		log.Printf("store gzipped (%d%%) data to %s...",
			len(zdata)*100/len(data), prc.StateFName)
		util.Store(prc.StateFName, zdata)
	} else {
		log.Printf("store ungzipped data to %s...", prc.StateFName)
		util.Store(prc.StateFName, data)
	}
}