示例#1
0
func DownloadHTMLData(url string) []byte {
	resp, err := http.Get(url)
	helper.HandleFatalError("error getting html: ", err)
	body, err := ioutil.ReadAll(resp.Body)
	helper.HandleFatalError("error reading html data: ", err)
	resp.Body.Close()
	return body
}
示例#2
0
func DownloadSeason(id string) []byte {
	params := url.Values{"filter_saison_id": []string{id}, "task": []string{"veranstaltungen"}}
	resp, err := http.PostForm("http://www.kickern-hamburg.de/liga-tool/mannschaftswettbewerbe", params)
	helper.HandleFatalError("error getting season: ", err)
	body, err := ioutil.ReadAll(resp.Body)
	helper.HandleFatalError("error reading season data: ", err)
	resp.Body.Close()
	return body
}
示例#3
0
func parseScores(scores string) (homeScore int, guestScore int) {
	var err error
	scoreChunks := strings.Split(scores, ":")
	homeScore, err = strconv.Atoi(strings.TrimSpace(scoreChunks[0]))
	helper.HandleFatalError("parsing home score failed:", err)
	guestScore, err = strconv.Atoi(strings.TrimSpace(scoreChunks[1]))
	helper.HandleFatalError("parsing guest score failed:", err)
	return
}
示例#4
0
func (this *CouchDbRepo) GetLatestMessageDate() time.Time {
	var result QueryResult

	err := this.getConnection().Query("_design/messages/_view/by_date", map[string]interface{}{"limit": 1, "descending": true}, &result)
	helper.HandleFatalError("getting latest message date failed:", err)
	if len(result.Rows) == 0 {
		return time.Date(1990, 01, 01, 0, 0, 0, 0, time.UTC)
	}
	unixtimestamp, err := strconv.ParseInt(result.Rows[0].Key, 10, 64)
	helper.HandleFatalError("parsing last date from db failed:", err)
	return time.Unix(unixtimestamp, 0)
}
示例#5
0
func (this *Webservice) Start() {
	tcpAddr, err := net.ResolveTCPAddr("tcp", ":"+strconv.Itoa(this.listenPort))
	helper.HandleFatalError("error resolving webservice address.", err)
	this.listener, err = net.ListenTCP("tcp", tcpAddr)
	helper.HandleFatalError("error opening webservice port.", err)

	handler := &Handler{ServeMux: http.NewServeMux()}

	handler.ServeMux.HandleFunc(this.listenPath,
		func(res http.ResponseWriter, req *http.Request) {
			this.feedHandler(res, req)
		})

	go http.Serve(this.listener, handler)
}
示例#6
0
func parseGamePosition(selection *goquery.Selection) (position int) {
	positionString := strings.TrimSpace(selection.Children().First().Text())
	var err error
	position, err = strconv.Atoi(strings.TrimSpace(positionString))
	helper.HandleFatalError("parsing game position failed:", err)
	return
}
示例#7
0
文件: parser.go 项目: sejoharp/hpfeed
func (this *Parser) GenerateDocument(rawData []byte) *goquery.Document {
	utf8String := toUtf8(rawData)
	utf8byteArray := []byte(utf8String)
	node, err := html.Parse(bytes.NewReader(utf8byteArray))
	helper.HandleFatalError("document generation failed:", err)
	return goquery.NewDocumentFromNode(node)
}
示例#8
0
func (this *ForumReader) logout(client *http.Client) {
	sessionId := getSessionId(client.Jar)
	logoutUrl := forumPrefix + "/login.php?logout=true&sid=" + sessionId
	resp, err := client.Get(logoutUrl)
	helper.HandleFatalError("logout error: ", err)
	resp.Body.Close()
}
示例#9
0
func (this *ForumReader) getHTMLData(client *http.Client) []byte {
	resp, err := client.Get(forumPrefix + "/viewforum.php?f=15")
	helper.HandleFatalError("reading data error: ", err)
	body, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	return body
}
示例#10
0
func (this *CouchDbRepo) StoreAll(messages []*DbMessage) {
	couchDbMessageList := convertAllMessagesToCouchDb(messages)
	for _, couchDbMessage := range couchDbMessageList {
		couchDbMessage.ContentType = "news"
		_, _, err := this.getConnection().Insert(couchDbMessage)
		helper.HandleFatalError("error inserting couchDbMessage:", err)
	}
}
示例#11
0
func (this *MongoDbNewsRepo) GetAllMessages() []*DbMessage {
	collection := this.getNewsCollection()
	defer collection.Database.Session.Close()

	result := make([]*News, 0)
	err := collection.Find(nil).Sort("-date").All(&result)
	helper.HandleFatalError("error getting all News:", err)
	return convertAllNewsToMesssages(result)
}
示例#12
0
func (this *MongoDbNewsRepo) StoreAll(messages []*DbMessage) {
	collection := this.getNewsCollection()
	defer collection.Database.Session.Close()
	newsList := convertAllMessagesToNews(messages)
	for _, news := range newsList {
		err := collection.Insert(news)
		helper.HandleFatalError("error inserting news:", err)
	}
}
示例#13
0
func (this *MongoDbNewsRepo) GetLatestMessageDate() time.Time {
	collection := this.getNewsCollection()
	defer collection.Database.Session.Close()

	result := News{}
	err := collection.Find(nil).Select(bson.M{"date": 1}).Sort("-date").One(&result)
	helper.HandleFatalError("error getting latest News:", err)
	return result.Date
}
示例#14
0
func (this *CouchDbRepo) GetAllMessages() []*DbMessage {
	var result QueryResult

	err := this.getConnection().Query("_design/messages/_view/by_date", map[string]interface{}{"include_docs": true, "descending": true}, &result)
	helper.HandleFatalError("query all messages failed:", err)

	messages := make([]*DbMessage, 0)
	for i, _ := range result.Rows {
		messages = append(messages, convertCouchDbToMessage(&result.Rows[i].Doc))
	}
	return messages
}
示例#15
0
func parseMatchDay(doc *goquery.Document, hasMatchDate bool) int {
	rawData := doc.Find("#Content table tbody > tr > td").First().Text()
	dateChunks := strings.Split(rawData, ",")
	var matchDayString string
	if hasMatchDate {
		matchDayString = strings.Split(dateChunks[2], ".")[0]
	} else {
		matchDayString = strings.Split(dateChunks[1], ".")[0]
	}
	matchDay, err := strconv.Atoi(strings.TrimSpace(matchDayString))
	helper.HandleFatalError("parsing matchday failed:", err)
	return matchDay
}
示例#16
0
func StoreGamesInCSVFile(fileName string, games []*Game) {
	os.Remove(fileName)
	file, err := os.Create(fileName)
	helper.HandleFatalError("error creating csv file: ", err)
	fileWriter := bufio.NewWriter(file)
	delimiter, _ := utf8.DecodeRuneInString(";")
	csvWriter := csv.NewWriter(fileWriter)
	csvWriter.Comma = delimiter
	for _, game := range games {
		if game.HomePlayer2 == "" {
			game.HomePlayer2 = "XXXX"
		}
		if game.GuestPlayer2 == "" {
			game.GuestPlayer2 = "XXXX"

		}
		record := []string{
			game.MatchDate.Format("2006-01-02"),
			strconv.Itoa(game.MatchDay),
			strconv.Itoa(game.Position),
			game.HomeTeam,
			game.HomePlayer1,
			game.HomePlayer2,
			strconv.Itoa(game.HomeScore),
			strconv.Itoa(game.GuestScore),
			game.GuestPlayer1,
			game.GuestPlayer2,
			game.GuestTeam}
		csvWriter.Write(record)
	}
	csvWriter.Flush()
	err = fileWriter.Flush()
	helper.HandleFatalError("error flushing file writer: ", err)
	err = file.Close()
	helper.HandleFatalError("error closing file: ", err)
}
示例#17
0
func (this *ForumReader) login(client *http.Client) {
	params := url.Values{"username": []string{this.forumUser}, "password": []string{this.forumPasswd}, "login": {"anmelden"}}
	resp, err := client.PostForm(forumPrefix+"/login.php", params)
	helper.HandleFatalError("login error: ", err)
	resp.Body.Close()
}
示例#18
0
func (this *MongoDbNewsRepo) getNewsCollection() *mgo.Collection {
	session, err := mgo.Dial(this.dbhost)
	helper.HandleFatalError("error getting connection to mongodb:", err)
	return session.DB(this.dbname).C("news")
}
示例#19
0
func (this *Configurator) LoadConfig() *Config {
	var configFileName string
	flag.StringVar(&configFileName, "config", "hpfeed.conf", "path to config file")
	flag.Parse()

	config := Config{}
	configFile, err := conf.ReadConfigFile(configFileName)
	helper.HandleFatalError("loading config file failed (-config= forgotten):", err)

	config.Updateinterval, err = configFile.GetInt("", "updateinterval")
	helper.HandleFatalError("updateinterval", err)
	config.ListenPort, err = configFile.GetInt("", "listenPort")
	helper.HandleFatalError("listenPort", err)
	config.ListenPath, err = configFile.GetString("", "listenPath")
	helper.HandleFatalError("listenPath", err)
	config.Dbhost, err = configFile.GetString("", "dbhost")
	helper.HandleFatalError("dbhost", err)
	config.Dbport, err = configFile.GetString("", "dbport")
	helper.HandleFatalError("dbport", err)
	config.Dbname, err = configFile.GetString("", "dbname")
	helper.HandleFatalError("dbname", err)
	config.Dbuser, err = configFile.GetString("", "dbuser")
	helper.HandleFatalError("dbuser", err)
	config.Dbpassword, err = configFile.GetString("", "dbpassword")
	helper.HandleFatalError("dbpassword", err)
	config.ForumUser, err = configFile.GetString("", "forumUser")
	helper.HandleFatalError("forumUser", err)
	config.ForumPasswd, err = configFile.GetString("", "forumPasswd")
	helper.HandleFatalError("forumPasswd", err)
	return &config
}
示例#20
0
func (this *CouchDbRepo) getConnection() *couch.Database {
	dburl := "http://" + this.dbuser + ":" + this.dbpassword + "@" + this.dbhost + ":" + this.dbport + "/" + this.dbname
	conn, err := couch.Connect(dburl)
	helper.HandleFatalError("db connection error:", err)
	return &conn
}
示例#21
0
func stringToInt64(raw string) int64 {
	result, err := strconv.ParseInt(raw, 10, 64)
	helper.HandleFatalError("parsing string to int64 failed:", err)
	return result
}