func parseFile(in string, out string) int {
content, err := os.Open(in)
fantasyfootball.HandleError(err)
defer content.Close()
//create outputfile, don't know exactly what to call it yet
file, err := ioutil.TempFile(testOutDir, "parsed_")
fantasyfootball.HandleError(err)
//parse the file
week, pos := parse(content, file)
defer os.Remove(file.Name())
defer file.Close()
//name the file based off it's contents
var weekString string
if week == CurrWeekPlaceholder {
weekString = CurrWeekString
} else {
weekString = fmt.Sprintf("%v", week)
}
actualName := fmt.Sprintf("%s/%s_%v.txt", testOutDir, pos, weekString)
newFile, err := os.Create(actualName)
fantasyfootball.HandleError(err)
defer newFile.Close()
file.Seek(0, 0)
_, err = io.Copy(newFile, file)
fantasyfootball.HandleError(err)
return week
}
func main() {
flag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
positions, err := fantasyfootball.ParsePositions(*positionsFlag, ",")
fantasyfootball.HandleError(err)
if len(positions) == 0 {
panic("No positions specified")
}
var scraper *fscraper.Scraper
switch *siteFlag {
case "accuscore":
scraper = fscraper.NewAccuscoreScraper(positions, *startWeekFlag, *endWeekFlag)
case "nfl":
scraper = fscraper.NewNflScraper(positions, *seasonFlag, *startWeekFlag, *endWeekFlag)
default:
panic(fmt.Sprintf("Unexpected value for flag \"--site\": \"%s\"", *siteFlag))
}
for page := range scraper.Scrape() {
var content io.ReadCloser = page.Content
defer content.Close()
file, err := os.Create(*dataDirFlag + "/" + page.Description + ".html")
fantasyfootball.HandleError(err)
defer file.Close()
io.Copy(file, content)
}
}
func parse(in io.Reader, out io.Writer) (week int, pos string) {
content, err := ioutil.ReadAll(in)
fantasyfootball.HandleError(err)
//extract position from page
posMatch := posRegex.FindSubmatch(content)
if posMatch == nil {
fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse position from file: %s", in)))
}
pos = string(posMatch[1])
//extract week from page
weekMatch := weekRegex.FindSubmatch(content)
if weekMatch == nil {
fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse week from file: %s", in)))
}
var weekString string
if parseWeekMatch := parseWeek.FindSubmatch(weekMatch[1]); parseWeekMatch != nil {
weekString = fmt.Sprintf("%s", parseWeekMatch[1])
} else {
weekString = CurrWeekString
}
//prepare csv output
csvWriter := csv.NewWriter(out)
defer csvWriter.Flush()
csvWriter.Comma = '\t'
header := headerRegex.Find(content)
headerColumn := dataRegex.FindAllSubmatch(header, -1)
csvHeader := make([]string, len(headerColumn))
for i, head := range headerColumn {
csvHeader[i] = fmt.Sprintf("%s", head[1])
}
csvWriter.Write(csvHeader)
//extract player info from page
rows := rowRegex.FindAll(content, -1)
for _, row := range rows {
columns := dataRegex.FindAllSubmatch(row, -1)
csvRow := make([]string, len(columns))
for i, column := range columns {
csvRow[i] = fmt.Sprintf("%s", column[1])
}
csvWriter.Write(csvRow)
}
weekInt, err := strconv.ParseInt(weekString, 10, 0)
if err != nil {
weekInt = CurrWeekPlaceholder
}
return int(weekInt), pos
}
func indexHandler(w http.ResponseWriter, r *http.Request) {
fileDS := fantasyfootball.NewFileDataSource(dataSource, *startWeekFlag, *endWeekFlag)
normDS := fantasyfootball.NewNormalizedDataSource(fileDS)
err := templates.ExecuteTemplate(w, "index.html", normDS)
if err != nil {
fmt.Printf("%v", err)
}
fantasyfootball.HandleError(err)
}
func (persister *FilePersister) Persist(data *ScraperOutput) {
file, err := os.Create(persister.dataDir + "/" + persister.keyGen(data))
fantasyfootball.HandleError(err)
defer file.Close()
io.Copy(file, data)
}
func (parser *AccuscoreParser) parseContent(in io.Reader, out chan<- PlayerRecord) {
content, err := ioutil.ReadAll(in)
fantasyfootball.HandleError(err)
//extract position from page
posMatch := parser.position.FindSubmatch(content)
if posMatch == nil {
fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse position from file: %s", in)))
}
pos := string(posMatch[1])
//extract week from page
weekMatch := parser.week.FindSubmatch(content)
if weekMatch == nil {
fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse week from file: %s", in)))
}
var weekString string
if parseWeekMatch := parser.parseWeek.FindSubmatch(weekMatch[1]); parseWeekMatch != nil {
weekString = fmt.Sprintf("%s", parseWeekMatch[1])
} else {
weekString = parser.currWeekString
}
weekInt, err := strconv.ParseInt(weekString, 10, 0)
if err != nil {
weekInt = CurrWeekPlaceholder
}
//extract header
header := parser.header.Find(content)
headerColumns := parser.data.FindAllSubmatch(header, -1)
headerMap := make(map[int]string)
for i, headMatch := range headerColumns {
head := headMatch[1]
headerMap[i] = fmt.Sprintf("%s", head)
}
//extract player info from page
rows := parser.row.FindAll(content, -1)
for _, row := range rows {
columns := parser.data.FindAllSubmatch(row, -1)
dataMap := make(map[string]string)
for i, column := range columns {
columnData := fmt.Sprintf("%s", column[1])
dataHeader := headerMap[i]
dataMap[dataHeader] = columnData
}
footbalPosition, err := fantasyfootball.ParsePosition(pos)
fantasyfootball.HandleError(err)
record := PlayerRecord{
Week: int(weekInt),
Position: footbalPosition,
Name: dataMap[parser.nameString],
Team: dataMap[parser.teamString],
Stats: make(map[string]float64),
}
delete(dataMap, parser.nameString)
delete(dataMap, parser.teamString)
for key, value := range dataMap {
parsedFloat, err := strconv.ParseFloat(value, 64)
fantasyfootball.HandleError(err)
record.Stats[key] = parsedFloat
}
out <- record
}
}
func main() {
//create channel of content from directory
content := make(chan io.ReadCloser)
go func(newContent chan<- io.ReadCloser) {
files, err := ioutil.ReadDir(inputDir)
fantasyfootball.HandleError(err)
var channelContentWg sync.WaitGroup
for _, file := range files {
if file.IsDir() {
continue
}
channelContentWg.Add(1)
go func(fileName string, addContent chan<- io.ReadCloser, wg *sync.WaitGroup) {
fileStream, err := os.Open(fileName)
fantasyfootball.HandleError(err)
addContent <- fileStream
wg.Done()
}(fmt.Sprintf("%v/%v", inputDir, file.Name()), newContent, &channelContentWg)
}
channelContentWg.Wait()
close(newContent)
}(content)
parser := NewAccuscoreParser()
playerRecordsChannel := parser.Parse(content)
fileMap := make(map[string]*ParseOutput)
minWeekResult :=
func(playerRecords <-chan PlayerRecord) int {
var minWeek int = math.MaxInt32
for record := range playerRecords {
if record.Week > 0 && record.Week < minWeek {
minWeek = record.Week
}
fileName := fmt.Sprintf("%v_%v.txt", record.Position.String(), record.Week)
playerFile := fileMap[fileName]
if playerFile == nil {
file, err := os.Create(fmt.Sprintf("%v/%v", testOutDir, fileName))
fantasyfootball.HandleError(err)
csvWriter := csv.NewWriter(file)
defer file.Close()
defer csvWriter.Flush()
csvWriter.Comma = '\t'
headers := make([]string, len(record.Stats))
csvHeaders := make([]string, len(record.Stats)+2)
csvHeaders[0] = name
csvHeaders[1] = team
idx := 0
for key := range record.Stats {
headers[idx] = key
csvHeaders[idx+2] = key
idx++
}
csvWriter.Write(csvHeaders)
playerFile = &ParseOutput{
writer: csvWriter,
headers: headers,
}
fileMap[fileName] = playerFile
}
line := make([]string, len(record.Stats)+2)
line[0] = record.Name
line[1] = record.Team
for i, v := range playerFile.headers {
line[i+2] = fmt.Sprintf("%v", record.Stats[v])
}
playerFile.writer.Write(line)
}
return minWeek
}(playerRecordsChannel)
//rename current week file
newMinWeek := fmt.Sprintf("%v", minWeekResult-1)
for fName := range fileMap {
newFilename := placeholderRegex.ReplaceAllString(fName, newMinWeek)
if newFilename != fName {
fullOldFilename := fmt.Sprintf("%v/%v", testOutDir, fName)
oldFile, err := os.Open(fullOldFilename)
fantasyfootball.HandleError(err)
newFile, err := os.Create(fmt.Sprintf("%v/%v", testOutDir, newFilename))
defer newFile.Close()
fantasyfootball.HandleError(err)
_, err = io.Copy(newFile, oldFile)
fantasyfootball.HandleError(err)
err = oldFile.Close()
fantasyfootball.HandleError(err)
err = os.Remove(fullOldFilename)
fantasyfootball.HandleError(err)
}
}
}
func fetch(url string) *http.Response {
res, err := http.Get(url)
fantasyfootball.HandleError(err)
return res
}
