Exemple #1
0
func parseFile(in string, out string) int {
	content, err := os.Open(in)
	fantasyfootball.HandleError(err)
	defer content.Close()

	//create outputfile, don't know exactly what to call it yet
	file, err := ioutil.TempFile(testOutDir, "parsed_")
	fantasyfootball.HandleError(err)

	//parse the file
	week, pos := parse(content, file)
	defer os.Remove(file.Name())
	defer file.Close()

	//name the file based off it's contents
	var weekString string
	if week == CurrWeekPlaceholder {
		weekString = CurrWeekString
	} else {
		weekString = fmt.Sprintf("%v", week)
	}
	actualName := fmt.Sprintf("%s/%s_%v.txt", testOutDir, pos, weekString)
	newFile, err := os.Create(actualName)
	fantasyfootball.HandleError(err)
	defer newFile.Close()
	file.Seek(0, 0)
	_, err = io.Copy(newFile, file)
	fantasyfootball.HandleError(err)

	return week
}
func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	positions, err := fantasyfootball.ParsePositions(*positionsFlag, ",")
	fantasyfootball.HandleError(err)
	if len(positions) == 0 {
		panic("No positions specified")
	}
	var scraper *fscraper.Scraper
	switch *siteFlag {
	case "accuscore":
		scraper = fscraper.NewAccuscoreScraper(positions, *startWeekFlag, *endWeekFlag)
	case "nfl":
		scraper = fscraper.NewNflScraper(positions, *seasonFlag, *startWeekFlag, *endWeekFlag)
	default:
		panic(fmt.Sprintf("Unexpected value for flag \"--site\": \"%s\"", *siteFlag))
	}
	for page := range scraper.Scrape() {
		var content io.ReadCloser = page.Content
		defer content.Close()

		file, err := os.Create(*dataDirFlag + "/" + page.Description + ".html")
		fantasyfootball.HandleError(err)
		defer file.Close()

		io.Copy(file, content)
	}
}
Exemple #3
0
func parse(in io.Reader, out io.Writer) (week int, pos string) {
	content, err := ioutil.ReadAll(in)
	fantasyfootball.HandleError(err)

	//extract position from page
	posMatch := posRegex.FindSubmatch(content)
	if posMatch == nil {
		fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse position from file: %s", in)))
	}
	pos = string(posMatch[1])

	//extract week from page
	weekMatch := weekRegex.FindSubmatch(content)
	if weekMatch == nil {
		fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse week from file: %s", in)))
	}
	var weekString string
	if parseWeekMatch := parseWeek.FindSubmatch(weekMatch[1]); parseWeekMatch != nil {
		weekString = fmt.Sprintf("%s", parseWeekMatch[1])
	} else {
		weekString = CurrWeekString
	}

	//prepare csv output
	csvWriter := csv.NewWriter(out)
	defer csvWriter.Flush()
	csvWriter.Comma = '\t'

	header := headerRegex.Find(content)
	headerColumn := dataRegex.FindAllSubmatch(header, -1)
	csvHeader := make([]string, len(headerColumn))
	for i, head := range headerColumn {
		csvHeader[i] = fmt.Sprintf("%s", head[1])
	}
	csvWriter.Write(csvHeader)
	//extract player info from page
	rows := rowRegex.FindAll(content, -1)
	for _, row := range rows {
		columns := dataRegex.FindAllSubmatch(row, -1)
		csvRow := make([]string, len(columns))
		for i, column := range columns {
			csvRow[i] = fmt.Sprintf("%s", column[1])
		}
		csvWriter.Write(csvRow)
	}
	weekInt, err := strconv.ParseInt(weekString, 10, 0)
	if err != nil {
		weekInt = CurrWeekPlaceholder
	}
	return int(weekInt), pos
}
func indexHandler(w http.ResponseWriter, r *http.Request) {
	fileDS := fantasyfootball.NewFileDataSource(dataSource, *startWeekFlag, *endWeekFlag)
	normDS := fantasyfootball.NewNormalizedDataSource(fileDS)
	err := templates.ExecuteTemplate(w, "index.html", normDS)
	if err != nil {
		fmt.Printf("%v", err)
	}
	fantasyfootball.HandleError(err)
}
func (persister *FilePersister) Persist(data *ScraperOutput) {
	file, err := os.Create(persister.dataDir + "/" + persister.keyGen(data))
	fantasyfootball.HandleError(err)
	defer file.Close()
	io.Copy(file, data)
}
Exemple #6
0
func (parser *AccuscoreParser) parseContent(in io.Reader, out chan<- PlayerRecord) {
	content, err := ioutil.ReadAll(in)
	fantasyfootball.HandleError(err)

	//extract position from page
	posMatch := parser.position.FindSubmatch(content)
	if posMatch == nil {
		fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse position from file: %s", in)))
	}
	pos := string(posMatch[1])

	//extract week from page
	weekMatch := parser.week.FindSubmatch(content)
	if weekMatch == nil {
		fantasyfootball.HandleError(errors.New(fmt.Sprintf("unable to parse week from file: %s", in)))
	}
	var weekString string
	if parseWeekMatch := parser.parseWeek.FindSubmatch(weekMatch[1]); parseWeekMatch != nil {
		weekString = fmt.Sprintf("%s", parseWeekMatch[1])
	} else {
		weekString = parser.currWeekString
	}
	weekInt, err := strconv.ParseInt(weekString, 10, 0)
	if err != nil {
		weekInt = CurrWeekPlaceholder
	}

	//extract header
	header := parser.header.Find(content)
	headerColumns := parser.data.FindAllSubmatch(header, -1)
	headerMap := make(map[int]string)
	for i, headMatch := range headerColumns {
		head := headMatch[1]
		headerMap[i] = fmt.Sprintf("%s", head)
	}

	//extract player info from page
	rows := parser.row.FindAll(content, -1)
	for _, row := range rows {
		columns := parser.data.FindAllSubmatch(row, -1)
		dataMap := make(map[string]string)
		for i, column := range columns {
			columnData := fmt.Sprintf("%s", column[1])
			dataHeader := headerMap[i]
			dataMap[dataHeader] = columnData
		}
		footbalPosition, err := fantasyfootball.ParsePosition(pos)
		fantasyfootball.HandleError(err)
		record := PlayerRecord{
			Week:     int(weekInt),
			Position: footbalPosition,
			Name:     dataMap[parser.nameString],
			Team:     dataMap[parser.teamString],
			Stats:    make(map[string]float64),
		}
		delete(dataMap, parser.nameString)
		delete(dataMap, parser.teamString)
		for key, value := range dataMap {
			parsedFloat, err := strconv.ParseFloat(value, 64)
			fantasyfootball.HandleError(err)
			record.Stats[key] = parsedFloat
		}
		out <- record
	}
}
Exemple #7
0
func main() {
	//create channel of content from directory
	content := make(chan io.ReadCloser)
	go func(newContent chan<- io.ReadCloser) {
		files, err := ioutil.ReadDir(inputDir)
		fantasyfootball.HandleError(err)
		var channelContentWg sync.WaitGroup
		for _, file := range files {
			if file.IsDir() {
				continue
			}
			channelContentWg.Add(1)
			go func(fileName string, addContent chan<- io.ReadCloser, wg *sync.WaitGroup) {
				fileStream, err := os.Open(fileName)
				fantasyfootball.HandleError(err)
				addContent <- fileStream
				wg.Done()
			}(fmt.Sprintf("%v/%v", inputDir, file.Name()), newContent, &channelContentWg)
		}
		channelContentWg.Wait()
		close(newContent)
	}(content)

	parser := NewAccuscoreParser()
	playerRecordsChannel := parser.Parse(content)

	fileMap := make(map[string]*ParseOutput)
	minWeekResult :=
		func(playerRecords <-chan PlayerRecord) int {
			var minWeek int = math.MaxInt32
			for record := range playerRecords {
				if record.Week > 0 && record.Week < minWeek {
					minWeek = record.Week
				}
				fileName := fmt.Sprintf("%v_%v.txt", record.Position.String(), record.Week)
				playerFile := fileMap[fileName]
				if playerFile == nil {
					file, err := os.Create(fmt.Sprintf("%v/%v", testOutDir, fileName))
					fantasyfootball.HandleError(err)
					csvWriter := csv.NewWriter(file)
					defer file.Close()
					defer csvWriter.Flush()
					csvWriter.Comma = '\t'
					headers := make([]string, len(record.Stats))
					csvHeaders := make([]string, len(record.Stats)+2)
					csvHeaders[0] = name
					csvHeaders[1] = team
					idx := 0
					for key := range record.Stats {
						headers[idx] = key
						csvHeaders[idx+2] = key
						idx++
					}
					csvWriter.Write(csvHeaders)
					playerFile = &ParseOutput{
						writer:  csvWriter,
						headers: headers,
					}
					fileMap[fileName] = playerFile
				}
				line := make([]string, len(record.Stats)+2)
				line[0] = record.Name
				line[1] = record.Team
				for i, v := range playerFile.headers {
					line[i+2] = fmt.Sprintf("%v", record.Stats[v])
				}
				playerFile.writer.Write(line)
			}
			return minWeek
		}(playerRecordsChannel)
	//rename current week file
	newMinWeek := fmt.Sprintf("%v", minWeekResult-1)
	for fName := range fileMap {
		newFilename := placeholderRegex.ReplaceAllString(fName, newMinWeek)
		if newFilename != fName {
			fullOldFilename := fmt.Sprintf("%v/%v", testOutDir, fName)
			oldFile, err := os.Open(fullOldFilename)
			fantasyfootball.HandleError(err)
			newFile, err := os.Create(fmt.Sprintf("%v/%v", testOutDir, newFilename))
			defer newFile.Close()
			fantasyfootball.HandleError(err)
			_, err = io.Copy(newFile, oldFile)
			fantasyfootball.HandleError(err)
			err = oldFile.Close()
			fantasyfootball.HandleError(err)
			err = os.Remove(fullOldFilename)
			fantasyfootball.HandleError(err)
		}
	}
}
Exemple #8
0
func fetch(url string) *http.Response {
	res, err := http.Get(url)
	fantasyfootball.HandleError(err)
	return res
}