func githubMarkup() string {
body := body("https://github.com/trending")
doc, _ := gokogiri.ParseHtml(body)
xpath := css.Convert("div.leaderboard-list-content", css.GLOBAL)
links, _ := doc.Search(xpath)
var newLinks []interface{}
for _, l := range links {
result := ""
repos, _ := l.Search(css.Convert(".repository-name", css.LOCAL))
desc, _ := l.Search(css.Convert(".repo-leaderboard-description", css.LOCAL))
if len(repos) > 0 {
absoluteURL(repos[0])
result += repos[0].String()
}
if len(desc) > 0 {
result += desc[0].String()
}
newLinks = append(newLinks, template.HTML(result))
}
tmpl, err := template.New("github-trending").Parse(`
<ul>
{{range .links}}
<li>{{.}}</li>
{{end}}
</ul>
`)
checkErr(err, "template error")
var results bytes.Buffer
tmpl.Execute(&results, map[string]interface{}{
"links": newLinks,
})
return string(results.Bytes())
}
func hackerNewsMarkup() string {
fmt.Println("Fetching hackernews.....")
body := body("https://news.ycombinator.com/")
doc, _ := gokogiri.ParseHtml(body)
xpath := css.Convert("td.title a", css.GLOBAL)
links, _ := doc.Search(xpath)
var newLinks []HNLink
var chanLinks = make(chan HNLink)
for _, l := range links[:len(links)-1] {
go parseLink(l, chanLinks)
}
// collect back links info
for i := 0; i < len(links)-1; i++ {
newLinks = append(newLinks, <-chanLinks)
}
tmpl, err := template.New("hackernews").Parse(`
<ul>
{{range .links}}
<li>{{.LinkMarkup}}</li>
<p>{{.Excerpt}}</p>
{{end}}
</ul>
`)
checkErr(err, "template error")
var results bytes.Buffer
tmpl.Execute(&results, map[string]interface{}{
"links": newLinks,
})
return string(results.Bytes())
}
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"github.com/kennygrant/sanitize"
"github.com/moovweb/gokogiri"
"github.com/moovweb/gokogiri/css"
)
var googleDocQuery string = css.Convert("#center_col > div > div > a > b > i", 0) + "/text()[1]"
type searchEngine struct {
name string
baseURL string
queryParam string
suggestedTermQuery string
topLinkQuery string
topTitleQuery string
}
type correctionResult struct {
engine string
SuggestedTerm string
TopLink string
TopTitle string
func toXpath(cssQuery string) string {
return gcss.Convert(cssQuery, gcss.LOCAL)
}