Example #1
0
func DoFetch(cf *config.Config) {
	log.Println("=== FETCH BEGIN ===")
	s := new(fetcher.Session)
	s.Config = cf
	for _, realm := range cf.RealmsList {
		for _, locale := range cf.LocalesList {
			file_url, file_ts := s.Fetch_FileURL(realm, locale)
			log.Printf("FILE URL: %s", file_url)
			log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
			fname := util.Make_FName(realm, file_ts, true)
			json_fname := cf.DownloadDirectory + fname
			if !util.CheckFile(json_fname) {
				log.Printf("downloading from %s ...", file_url)
				data := s.Get(file_url)
				log.Printf("... got %d octets", len(data))
				zdata := util.Zip(data)
				log.Printf("... zipped to %d octets (%d%%)",
					len(zdata), len(zdata)*100/len(data))
				util.Store(json_fname, zdata)
				log.Printf("stored to %s .", json_fname)
			} else {
				log.Println("... already downloaded")
			}
		}
	}
	log.Println("=== FETCH END ===")
}
Example #2
0
func ParseDir(cf *config.Config, realm string, safe bool) {
	mask := cf.DownloadDirectory +
		strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
	log.Printf("scan by mask %s ...", mask)
	fnames, err := filepath.Glob(mask)
	if err != nil {
		log.Fatalln("glob failed:", err)
	}
	log.Printf("... %d entries collected", len(fnames))

	var goodfnames []string

	for _, fname := range fnames {
		// realm, ts, good := util.Parse_FName(fname)
		_, _, good := util.Parse_FName(fname)
		if good {
			// log.Printf("fname %s -> %s, %v", fname, realm, ts)
			goodfnames = append(goodfnames, fname)
		} else {
			// log.Printf("skip fname %s", fname)
		}
	}
	sort.Sort(util.ByBasename(goodfnames))
	prc := new(AuctionProcessor)
	prc.Init(cf, realm)
	prc.LoadState()
	badfiles := make(map[string]string)

	for _, fname := range fnames {
		//log.Println(fname)
		f_realm, f_time, ok := util.Parse_FName(fname)
		if !ok {
			log.Fatalf("not parsed correctly: %s", fname)
			continue
		}
		if f_realm != realm {
			log.Fatalf("not my realm (%s != %s)", f_realm, realm)
			continue
		}
		if !prc.SnapshotNeeded(f_time) {
			log.Printf("snapshot not needed: %s", util.TSStr(f_time))
			continue
		}
		data, err := util.Load(fname)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s LOAD ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}
		ss, err := ParseSnapshot(data)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s PARSE ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}

		prc.StartSnapshot(f_time)
		for _, auc := range ss.Auctions {
			prc.AddAuctionEntry(&auc)
		}
		prc.FinishSnapshot()
		if safe {
			prc.SaveState()
		}
	}
	if !safe {
		prc.SaveState()
	}
	if len(badfiles) == 0 {
		log.Printf("all files loaded without errors")
	} else {
		log.Printf("%d files with errors", len(badfiles))
		for fname, err := range badfiles {
			log.Printf("%s: %s", fname, err)
		}
	}
}
Example #3
0
func (prc *AuctionProcessor) FinishSnapshot() {
	if !prc.Started {
		log.Panic("FinishSnapshot outside snapshot session")
	}

	// log.Println("check for closed auctions")
	num_open, num_closed := 0, 0
	auc_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("auctions", prc.Realm, prc.SnapshotTime)
	meta_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("metadata", prc.Realm, prc.SnapshotTime)
	snap_fname := prc.cf.ResultDirectory + prc.cf.GetTimedName("snapshot", prc.Realm, prc.SnapshotTime)

	prc.FileAuc = OpenOrCreateFile(auc_fname)
	defer prc.FileAuc.Close()

	prc.FileMeta = OpenOrCreateFile(meta_fname)
	defer prc.FileMeta.Close()

	SnapInfo := OpenOrCreateFile(snap_fname)
	defer SnapInfo.Close()

	for id, _ := range prc.State.WorkSet {
		_, seen := prc.SeenSet[id]
		if !seen {
			num_closed++
			prc.closeEntry(id)
		} else {
			num_open++
		}
	}

	var rate int = 0
	if num_closed > 0 {
		rate = (prc.NumBought + prc.NumAuctioned) * 100 / num_closed
	}

	prc.TotalOpened += prc.NumCreated
	prc.TotalClosed += num_closed
	prc.TotalSuccess += prc.NumBought + prc.NumAuctioned
	var total_rate int = 0
	if prc.TotalClosed > 0 {
		total_rate = prc.TotalSuccess * 100 / prc.TotalClosed
	}

	log.Printf("%s: \n"+
		"    entries: %d\n"+
		"    active: %d,\n"+
		"    created: %d,\n"+
		"    changed: %d [bids: %d, adj: %d, moves: %d]\n"+
		"    closed: %d [bought: %d, auctioned: %d, expired: %d, succes: %d%%]",
		util.TSStr(prc.SnapshotTime),
		len(prc.State.WorkSet), num_open,
		prc.NumCreated, prc.NumModified,
		prc.NumBids, prc.NumAdjusts, prc.NumMoves,
		num_closed, prc.NumBought, prc.NumAuctioned, prc.NumExpired, rate)

	log.Printf("total created %d, closed %d, success %d%%",
		prc.TotalOpened, prc.TotalClosed, total_rate)

	SnapInfo.WriteString(
		fmt.Sprintf("%s: entries:%d  active:%d created:%d "+
			"changed:%d [bids:%d adj:%d moves:%d] "+
			"closed:%d [bought:%d auctioned:%d expired:%d rate:%d%%]\n",
			util.TSStr(prc.SnapshotTime),
			len(prc.State.WorkSet), num_open,
			prc.NumCreated, prc.NumModified,
			prc.NumBids, prc.NumAdjusts, prc.NumMoves,
			num_closed, prc.NumBought, prc.NumAuctioned, prc.NumExpired,
			rate))

	prc.State.LastTime = prc.SnapshotTime
	//log.Printf("last time sets to %s", util.TSStr(prc.State.LastTime))

	prc.Started = false
}