func DownloadHTMLData(url string) []byte {
resp, err := http.Get(url)
helper.HandleFatalError("error getting html: ", err)
body, err := ioutil.ReadAll(resp.Body)
helper.HandleFatalError("error reading html data: ", err)
resp.Body.Close()
return body
}
func DownloadSeason(id string) []byte {
params := url.Values{"filter_saison_id": []string{id}, "task": []string{"veranstaltungen"}}
resp, err := http.PostForm("http://www.kickern-hamburg.de/liga-tool/mannschaftswettbewerbe", params)
helper.HandleFatalError("error getting season: ", err)
body, err := ioutil.ReadAll(resp.Body)
helper.HandleFatalError("error reading season data: ", err)
resp.Body.Close()
return body
}
func parseScores(scores string) (homeScore int, guestScore int) {
var err error
scoreChunks := strings.Split(scores, ":")
homeScore, err = strconv.Atoi(strings.TrimSpace(scoreChunks[0]))
helper.HandleFatalError("parsing home score failed:", err)
guestScore, err = strconv.Atoi(strings.TrimSpace(scoreChunks[1]))
helper.HandleFatalError("parsing guest score failed:", err)
return
}
func (this *CouchDbRepo) GetLatestMessageDate() time.Time {
var result QueryResult
err := this.getConnection().Query("_design/messages/_view/by_date", map[string]interface{}{"limit": 1, "descending": true}, &result)
helper.HandleFatalError("getting latest message date failed:", err)
if len(result.Rows) == 0 {
return time.Date(1990, 01, 01, 0, 0, 0, 0, time.UTC)
}
unixtimestamp, err := strconv.ParseInt(result.Rows[0].Key, 10, 64)
helper.HandleFatalError("parsing last date from db failed:", err)
return time.Unix(unixtimestamp, 0)
}
func (this *Webservice) Start() {
tcpAddr, err := net.ResolveTCPAddr("tcp", ":"+strconv.Itoa(this.listenPort))
helper.HandleFatalError("error resolving webservice address.", err)
this.listener, err = net.ListenTCP("tcp", tcpAddr)
helper.HandleFatalError("error opening webservice port.", err)
handler := &Handler{ServeMux: http.NewServeMux()}
handler.ServeMux.HandleFunc(this.listenPath,
func(res http.ResponseWriter, req *http.Request) {
this.feedHandler(res, req)
})
go http.Serve(this.listener, handler)
}
func parseGamePosition(selection *goquery.Selection) (position int) {
positionString := strings.TrimSpace(selection.Children().First().Text())
var err error
position, err = strconv.Atoi(strings.TrimSpace(positionString))
helper.HandleFatalError("parsing game position failed:", err)
return
}
func (this *Parser) GenerateDocument(rawData []byte) *goquery.Document {
utf8String := toUtf8(rawData)
utf8byteArray := []byte(utf8String)
node, err := html.Parse(bytes.NewReader(utf8byteArray))
helper.HandleFatalError("document generation failed:", err)
return goquery.NewDocumentFromNode(node)
}
func (this *ForumReader) logout(client *http.Client) {
sessionId := getSessionId(client.Jar)
logoutUrl := forumPrefix + "/login.php?logout=true&sid=" + sessionId
resp, err := client.Get(logoutUrl)
helper.HandleFatalError("logout error: ", err)
resp.Body.Close()
}
func (this *ForumReader) getHTMLData(client *http.Client) []byte {
resp, err := client.Get(forumPrefix + "/viewforum.php?f=15")
helper.HandleFatalError("reading data error: ", err)
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
return body
}
func (this *CouchDbRepo) StoreAll(messages []*DbMessage) {
couchDbMessageList := convertAllMessagesToCouchDb(messages)
for _, couchDbMessage := range couchDbMessageList {
couchDbMessage.ContentType = "news"
_, _, err := this.getConnection().Insert(couchDbMessage)
helper.HandleFatalError("error inserting couchDbMessage:", err)
}
}
func (this *MongoDbNewsRepo) GetAllMessages() []*DbMessage {
collection := this.getNewsCollection()
defer collection.Database.Session.Close()
result := make([]*News, 0)
err := collection.Find(nil).Sort("-date").All(&result)
helper.HandleFatalError("error getting all News:", err)
return convertAllNewsToMesssages(result)
}
func (this *MongoDbNewsRepo) StoreAll(messages []*DbMessage) {
collection := this.getNewsCollection()
defer collection.Database.Session.Close()
newsList := convertAllMessagesToNews(messages)
for _, news := range newsList {
err := collection.Insert(news)
helper.HandleFatalError("error inserting news:", err)
}
}
func (this *MongoDbNewsRepo) GetLatestMessageDate() time.Time {
collection := this.getNewsCollection()
defer collection.Database.Session.Close()
result := News{}
err := collection.Find(nil).Select(bson.M{"date": 1}).Sort("-date").One(&result)
helper.HandleFatalError("error getting latest News:", err)
return result.Date
}
func (this *CouchDbRepo) GetAllMessages() []*DbMessage {
var result QueryResult
err := this.getConnection().Query("_design/messages/_view/by_date", map[string]interface{}{"include_docs": true, "descending": true}, &result)
helper.HandleFatalError("query all messages failed:", err)
messages := make([]*DbMessage, 0)
for i, _ := range result.Rows {
messages = append(messages, convertCouchDbToMessage(&result.Rows[i].Doc))
}
return messages
}
func parseMatchDay(doc *goquery.Document, hasMatchDate bool) int {
rawData := doc.Find("#Content table tbody > tr > td").First().Text()
dateChunks := strings.Split(rawData, ",")
var matchDayString string
if hasMatchDate {
matchDayString = strings.Split(dateChunks[2], ".")[0]
} else {
matchDayString = strings.Split(dateChunks[1], ".")[0]
}
matchDay, err := strconv.Atoi(strings.TrimSpace(matchDayString))
helper.HandleFatalError("parsing matchday failed:", err)
return matchDay
}
func StoreGamesInCSVFile(fileName string, games []*Game) {
os.Remove(fileName)
file, err := os.Create(fileName)
helper.HandleFatalError("error creating csv file: ", err)
fileWriter := bufio.NewWriter(file)
delimiter, _ := utf8.DecodeRuneInString(";")
csvWriter := csv.NewWriter(fileWriter)
csvWriter.Comma = delimiter
for _, game := range games {
if game.HomePlayer2 == "" {
game.HomePlayer2 = "XXXX"
}
if game.GuestPlayer2 == "" {
game.GuestPlayer2 = "XXXX"
}
record := []string{
game.MatchDate.Format("2006-01-02"),
strconv.Itoa(game.MatchDay),
strconv.Itoa(game.Position),
game.HomeTeam,
game.HomePlayer1,
game.HomePlayer2,
strconv.Itoa(game.HomeScore),
strconv.Itoa(game.GuestScore),
game.GuestPlayer1,
game.GuestPlayer2,
game.GuestTeam}
csvWriter.Write(record)
}
csvWriter.Flush()
err = fileWriter.Flush()
helper.HandleFatalError("error flushing file writer: ", err)
err = file.Close()
helper.HandleFatalError("error closing file: ", err)
}
func (this *ForumReader) login(client *http.Client) {
params := url.Values{"username": []string{this.forumUser}, "password": []string{this.forumPasswd}, "login": {"anmelden"}}
resp, err := client.PostForm(forumPrefix+"/login.php", params)
helper.HandleFatalError("login error: ", err)
resp.Body.Close()
}
func (this *MongoDbNewsRepo) getNewsCollection() *mgo.Collection {
session, err := mgo.Dial(this.dbhost)
helper.HandleFatalError("error getting connection to mongodb:", err)
return session.DB(this.dbname).C("news")
}
func (this *Configurator) LoadConfig() *Config {
var configFileName string
flag.StringVar(&configFileName, "config", "hpfeed.conf", "path to config file")
flag.Parse()
config := Config{}
configFile, err := conf.ReadConfigFile(configFileName)
helper.HandleFatalError("loading config file failed (-config= forgotten):", err)
config.Updateinterval, err = configFile.GetInt("", "updateinterval")
helper.HandleFatalError("updateinterval", err)
config.ListenPort, err = configFile.GetInt("", "listenPort")
helper.HandleFatalError("listenPort", err)
config.ListenPath, err = configFile.GetString("", "listenPath")
helper.HandleFatalError("listenPath", err)
config.Dbhost, err = configFile.GetString("", "dbhost")
helper.HandleFatalError("dbhost", err)
config.Dbport, err = configFile.GetString("", "dbport")
helper.HandleFatalError("dbport", err)
config.Dbname, err = configFile.GetString("", "dbname")
helper.HandleFatalError("dbname", err)
config.Dbuser, err = configFile.GetString("", "dbuser")
helper.HandleFatalError("dbuser", err)
config.Dbpassword, err = configFile.GetString("", "dbpassword")
helper.HandleFatalError("dbpassword", err)
config.ForumUser, err = configFile.GetString("", "forumUser")
helper.HandleFatalError("forumUser", err)
config.ForumPasswd, err = configFile.GetString("", "forumPasswd")
helper.HandleFatalError("forumPasswd", err)
return &config
}
func (this *CouchDbRepo) getConnection() *couch.Database {
dburl := "http://" + this.dbuser + ":" + this.dbpassword + "@" + this.dbhost + ":" + this.dbport + "/" + this.dbname
conn, err := couch.Connect(dburl)
helper.HandleFatalError("db connection error:", err)
return &conn
}
func stringToInt64(raw string) int64 {
result, err := strconv.ParseInt(raw, 10, 64)
helper.HandleFatalError("parsing string to int64 failed:", err)
return result
}
