func TestSubmittingAComment(t *testing.T) {
// Given a server
_, server, _, err := LoginUser(t)
if err != nil {
t.Fatal(err)
}
// When an authenticated client requests /item?id=1
id := 1
res, err := http.Get(fmt.Sprintf(server.URL+"/item?id=%d", id))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("status : want '%v' got '%v' ", want, got)
}
doc, err := goquery.NewDocumentFromResponse(res)
csrf, ok := doc.Find("input[name='comment_csrf']").First().Attr("value")
if !ok {
t.Fatalf("csrf value not found for comment form")
}
initialCommentNumber := doc.Find(".comment").Length()
formValues := url.Values{
"comment_content": {"this is a new comment"},
"comment_csrf": {csrf},
"comment_submit": {"submit"},
"comment_parent_id": {"0"},
"comment_goto": {fmt.Sprintf("/item?id=%d", id)},
"comment_thread_id": {fmt.Sprintf("%d", id)},
}
// when an authenicated client submits a valid comment
res, err = http.Post(server.URL+gonews.Route{}.Reply(), "application/x-www-form-urlencoded", strings.NewReader(formValues.Encode()))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("Status : want '%v' got '%v' ", want, got)
}
// The number of comments on the story page should have increased by one
doc, err = goquery.NewDocumentFromResponse(res)
if err != nil {
t.Fatal(err)
}
if want, got := initialCommentNumber+1, doc.Find(".comment").Length(); want != got {
t.Fatalf(".comment length : want '%v' got '%v' ", want, got)
}
}
func main() {
login()
for _, v := range cookies {
fmt.Println(v)
}
var bash_str string = "#!/bin/bash \n"
baseMap := map[string]string{}
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(spider_base_url))
doc.Find(".lesson-info-h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
base_href, _ := selection.Attr("href")
dir_name := selection.Text()
bash_str += "mkdir \"" + dir_name + "\"\n"
baseMap[dir_name] = base_href
fmt.Println(dir_name, "-->", base_href)
})
downloadList := []DownloadBean{}
for k, v := range baseMap {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v))
doc.Find(".lessonvideo-list dd h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
href, _ := selection.Attr("href")
filename := selection.Text()
fmt.Println(k, "-->", filename, "-->", href)
bean := DownloadBean{dirname: k, href: href, filename: filename}
downloadList = append(downloadList, bean)
})
}
for _, v := range downloadList {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v.href))
doc.Find("source").Each(func(i int, contentSelection *goquery.Selection) {
download_url, _ := contentSelection.Attr("src")
one_file := "wget " + download_url + " -O \"./" + v.dirname + "/" + v.filename + ".mp4\"\n"
bash_str += one_file
fmt.Println(one_file)
})
}
err := ioutil.WriteFile("./download.sh", []byte(bash_str), 0x777)
if err != nil {
panic(err.Error())
}
fmt.Println("写入脚本完成")
}
// GetResources retrieves resource info
func GetResources(resp *http.Response) Resources {
doc, _ := goquery.NewDocumentFromResponse(resp)
resWrapperSelector := doc.Find("div#resWrap")
lumber := resWrapperSelector.Find("td#l4")
clay := resWrapperSelector.Find("td#l3")
iron := resWrapperSelector.Find("td#l2")
crop := resWrapperSelector.Find("td#l1")
lumberProd, lumberStored, lumberCapa := parseResource(lumber)
clayProd, clayStored, _ := parseResource(clay)
ironProd, ironStored, _ := parseResource(iron)
cropProd, cropStored, cropCapa := parseResource(crop)
return Resources{
Granary{
Capacity: cropCapa,
Crop: cropStored,
},
Warehouse{
Capacity: lumberCapa,
Lumber: lumberStored,
Clay: clayStored,
Iron: ironStored,
},
Production{
lumberProd,
clayProd,
ironProd,
cropProd,
},
}
}
//解析函数定义，声明必须相同，下同
func Parser(httpRes *http.Response) ([]string, []basic.Item) {
//两个需要返回的列表
linklist := make([]string, 0) //下一步需要请求的链接
itemlist := make([]basic.Item, 0) //保存的数据类型为 map[string]interface{}
//自定义部分
//抓取所有链接
doc, _ := goquery.NewDocumentFromResponse(httpRes)
doc.Find("a").Each(func(i int, s *goquery.Selection) {
link, exits := s.Attr("href")
if exits {
link = basic.CheckLink(link)
if link != "" {
linklist = append(linklist, link)
}
}
})
//保存每个页面的标题
title := strings.TrimSpace(doc.Find("head title").Text())
if title != "" {
item := make(map[string]interface{})
item["标题"] = title
itemlist = append(itemlist, item)
}
return linklist, itemlist
}
func (api *ExternalAPIEngine) DoRequest(
arg APIArg, req *http.Request, restype int) (
ar *ExternalAPIRes, hr *ExternalHTMLRes, tr *ExternalTextRes, err error) {
var resp *http.Response
var jw *jsonw.Wrapper
resp, jw, err = doRequestShared(api, arg, req, (restype == XAPIResJSON))
if err != nil {
return
}
switch restype {
case XAPIResJSON:
ar = &ExternalAPIRes{resp.StatusCode, jw}
case XAPIResHTML:
var goq *goquery.Document
goq, err = goquery.NewDocumentFromResponse(resp)
if err == nil {
hr = &ExternalHTMLRes{resp.StatusCode, goq}
}
case XAPIResText:
var buf bytes.Buffer
_, err := buf.ReadFrom(resp.Body)
defer resp.Body.Close()
if err == nil {
tr = &ExternalTextRes{resp.StatusCode, string(buf.Bytes())}
}
default:
err = fmt.Errorf("unknown restype to DoRequest")
}
return
}
func harvest_board_indices(board_url string, board_name string) []BoardIndexPage {
var ret []BoardIndexPage
doc, err := goquery.NewDocumentFromResponse(ptt_get(board_url))
if err != nil {
log.Fatal(err)
}
re := regexp.MustCompile("/bbs/" + board_name + "/index([0-9]+)\\.html")
doc.Find("a[href^='/bbs/" + board_name + "/index']").Each(func(_ int, s *goquery.Selection) {
href, exists := s.Attr("href")
if !exists {
return
}
matched := re.FindStringSubmatch(href)
if len(matched) == 0 {
return
}
pn, err := strconv.Atoi(matched[1])
if err != nil {
log.Fatal(err)
}
ret = append(ret, BoardIndexPage{pn, href})
})
if ret[0].page_number > ret[1].page_number {
ret[0], ret[1] = ret[1], ret[0]
}
for i := ret[0].page_number + 1; i < ret[1].page_number; i++ {
ret = append(ret, BoardIndexPage{i, "/bbs/" + board_name + "/index" + strconv.Itoa(i) + ".html"})
}
return ret
}
func main() {
flag.Parse()
if flag.NArg() != 1 {
fmt.Fprintln(os.Stderr, "yomikata [word]")
os.Exit(1)
}
word := flag.Arg(0)
resp, err := http.Get("http://yomikata.org/word/" + word)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
if doc.Find("#word").Text() == "" {
fmt.Fprintln(os.Stderr, "わかりません")
os.Exit(1)
}
if *exact {
fmt.Println(doc.Find(".spAns .psAns").First().Text())
} else {
doc.Find(".spAns").Each(func(_ int, s *goquery.Selection) {
fmt.Printf("%s (%s)\n", s.Find(".psAns").Text(), s.Find(".psPt").Text())
})
}
}
// GetStatistics retrieves statistics
func GetStatistics(resp *http.Response) Statistics {
doc, _ := goquery.NewDocumentFromResponse(resp)
s := doc.Find("tr.hl").First()
positionStr := strings.TrimRight(s.Find(".ra").Text(), ".")
position, _ := strconv.Atoi(positionStr)
username := strings.Trim(s.Find(".pla").Find("a").Text(), " ")
alliance := s.Find("al").Text()
popStr := s.Find(".pop").Text()
pop, _ := strconv.Atoi(popStr)
villageCountStr := s.Find(".vil").Text()
villageCount, _ := strconv.Atoi(villageCountStr)
stats := Statistics{
position,
username,
alliance,
pop,
villageCount,
}
return stats
}
// FetchRzrqSumData 抓取数据
func FetchCwzb(code, ctype string) ([]*Cwzb, error) {
ctypeStr := "K"
if ctype == "sz" {
ctypeStr = "J"
}
formt := "http://quotes.cnfol.com/new/f10/cwzb/%s%s.html"
resp, err := wget.Get(fmt.Sprintf(formt, code, ctypeStr))
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
doc.Find("table tr").Each(func(i int, s *goquery.Selection) {
td := s.Find("td")
if td.Length() < 5 {
return
}
td.Each(func(i int, s *goquery.Selection) {
if i == 0 {
return
}
fmt.Println(s.Text())
})
})
return nil, nil
}
func mustParseResponse(resp *http.Response) *goquery.Document {
gq, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
log.Fatal(err)
}
return gq
}
func (a PlayStore) Search(sr SearchRequest) ([]App, error) {
playurl, post := a.searchUrl(sr)
body := strings.NewReader(post)
req, err := http.NewRequest("POST", playurl, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")
client := sr.Client
resp, err := client.Do(req)
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
cards := doc.Find(".card")
apps := make([]App, cards.Length())
cards.Each(func(i int, s *goquery.Selection) {
if val, ok := s.Attr("data-docid"); ok {
apps[i].Package = val
}
if val, ok := s.Find(".title").Attr("title"); ok {
apps[i].Name = val
}
if val, ok := s.Find(".subtitle").Attr("title"); ok {
apps[i].PublisherName = val
}
if val, ok := s.Find(".card-click-target").Attr("href"); ok {
apps[i].StoreUrl = "https://play.google.com" + val
}
if val, ok := s.Find(".cover-image").Attr("src"); ok {
apps[i].IconUrl = val
}
if val, ok := s.Find(".current-rating").Attr("style"); ok {
rating := strings.Split(val, " ")
if len(rating) >= 2 {
if rating_perc, err := strconv.ParseFloat(strings.TrimRight(rating[1], "%;"), 32); err == nil {
apps[i].Rating = float32(rating_perc * 5.0 / 100.0)
}
}
}
})
return apps, nil
}
// Get the collection of the ProxyList by parse the http response
func (this *XiCiDaiLi) Result(response *http.Response) ([]contract.ProxyList, error) {
doc, err := goquery.NewDocumentFromResponse(response)
if err != nil {
return nil, err
}
var proxyList []contract.ProxyList
proxyTable := doc.Find("table#ip_list")
proxyRows := proxyTable.Find("tr:nth-child(n+2)")
proxyRows.Each(func(i int, s *goquery.Selection) {
country := s.Find("td:nth-child(2) img").AttrOr("alt", "n/a")
ip := s.Find("td:nth-child(3)").Text()
port := s.Find("td:nth-child(4)").Text()
protocol := s.Find("td:nth-child(7)").Text()
if ip != "" && port != "" {
proxyList = append(proxyList, contract.ProxyList{
Ip: ip,
Port: port,
Protocol: protocol,
Country: strings.ToLower(country),
})
}
})
return proxyList, nil
}
func GetMajorList(client *http.Client) []string {
banlist := map[string]string{"110431": "", "110432": "", "110511": "", "110521": "", "113110": "", "113111": ""}
getmajorlist := []string{}
res, err := client.Get(DHUHostUrl + DHUGetMajorURL)
if err != nil {
fmt.Println(err)
fmt.Println("Something Wrong In GetMajorList")
} else {
doc, _ := goquery.NewDocumentFromResponse(res)
doc.Find("select").Eq(1).Find("option").Each(func(i int, s *goquery.Selection) {
sss, ok := s.Attr("value")
if ok {
_, testmap := banlist[sss]
if testmap {
return
} else {
getmajorlist = append(getmajorlist, sss)
}
} else {
fmt.Println("Something Wrong")
}
})
return getmajorlist
}
return nil
}
func GetAllLessons(res *http.Response) []CourseList {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
fmt.Println(err)
return nil
} else {
newlist := []CourseList{}
dec := mahonia.NewDecoder("GB18030")
doc.Find("tr").Each(func(i int, s *goquery.Selection) {
selectid := s.Find("td").Eq(0).Text()
_, err := strconv.Atoi(selectid)
if err == nil {
_, teachername, _ := dec.Translate([]byte(s.Find("td").Eq(6).Text()), true)
newinfo := []CourseInfo{}
s.Find("td").Eq(7).Find("tr").Each(func(i int, s *goquery.Selection) {
_, weektime, _ := dec.Translate([]byte(s.Find("td").Eq(0).Text()), true)
_, daytime, _ := dec.Translate([]byte(s.Find("td").Eq(1).Text()), true)
info := CourseInfo{string(weektime), string(daytime)}
newinfo = append(newinfo, info)
})
courselist := CourseList{selectid, string(teachername), newinfo}
newlist = append(newlist, courselist)
} else {
return
}
})
return newlist
}
}
func (this *Object) getHtml() (string, error) {
res, err := request("GET", this.TargetUrl, "")
if err != nil {
return "", err
}
if res.StatusCode != 200 {
return "", errors.New(fmt.Sprintf("http status not 200, it is %d", res.StatusCode))
}
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
return "", err
}
if err != nil {
return "", err
}
html, err := this.parse(doc)
if err != nil {
return "", err
}
this.setArgs(doc)
if err := this.setPageLimit(doc); err != nil {
return "", err
}
return html, nil
}
func (c Boot24Crawler) ParsePage(res *http.Response) *goquery.Document {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
log.Fatal(err)
}
return doc
}
// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
// http://stockData.stock.hexun.com/600028.shtml
formt := "http://stockData.stock.hexun.com/%s.shtml"
resp, err := wget.Get(fmt.Sprintf(formt, code))
if err != nil {
return nil, nil, gos.DoError(err)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, nil, gos.DoError(err)
}
tr := doc.Find("#list3 table.box6 tr")
stockIndustry := &StockIndustry{}
stockICB := &StockICB{}
stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
if err != nil {
return nil, nil, gos.DoError(err)
}
return stockIndustry, stockICB, nil
}
func get_site(uri string, r *http.Request) ([]temp_item, error) {
c := appengine.NewContext(r)
client := urlfetch.Client(c)
resp, err := client.Get(uri)
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
var list_items []temp_item
doc.Find("#bodyContent li").Not("#bodyContent #toc li").Each(func(i int, s *goquery.Selection) {
listitem := s.Text()
c.Infof("ITEM:\n" + listitem + "\n\n")
q, err := filter_question(listitem)
c.Infof("%v", err)
if err == nil {
list_items = append(list_items, q)
}
})
//c.Infof("%+v\n", list_items)
return list_items, nil
}
func GetRecipeIds(categoryId int, page int) ([]int, error) {
url := GetCategoryPageURL(categoryId, page)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to Get %s: %v", url, err)
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, fmt.Errorf("failed to parse response: %v", err)
}
recipeIds := []int{}
doc.Find(".info").Each(func(i int, s *goquery.Selection) {
s.Find(".name").Each(func(i int, s *goquery.Selection) {
if href, ok := s.Find("a").Attr("href"); ok {
id, err := strconv.Atoi(strings.Split(href, "/")[2])
if err != nil {
glog.Error("Failed to extract recipe id: %v", err)
} else {
recipeIds = append(recipeIds, id)
}
}
})
})
return recipeIds, nil
}
// BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
func BingDomain(domain, server string) (string, Results, error) {
task := "bing"
results := Results{}
resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
if err != nil {
return task, results, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return task, results, err
}
doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
u, err := url.Parse(s.Text())
if err != nil || u.Host == "" {
return
}
ip, err := LookupName(u.Host, server)
if err != nil || ip == "" {
cfqdn, err := LookupCname(u.Host, server)
if err != nil || cfqdn == "" {
return
}
ip, err = LookupName(cfqdn, server)
if err != nil || ip == "" {
return
}
}
results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
})
return task, results, err
}
func GetBuildings(resp *http.Response) Village {
doc, _ := goquery.NewDocumentFromResponse(resp)
village := Village{}
doc.Find("map#map2 area").EachWithBreak(func(index int, s *goquery.Selection) bool {
parcelId := getIdFromURL(s.AttrOr("href", "fail"))
name, level, empty := parseNameAndLevel(s.AttrOr("alt", "fail"))
if empty {
village.Parcels = append(village.Parcels, Parcel{
Id: parcelId,
Empty: true,
})
} else {
village.Parcels = append(village.Parcels, Parcel{
Id: parcelId,
Empty: false,
Building: Building{
Name: name,
Level: level,
},
})
}
if parcelId == 40 { // 40 = Wall; it is repeated 3 times, this limits it to one
return false
} else {
return true
}
})
return village
}
func (d *DHUStruct) ValidateStuCourseSelected(courseAndLesson map[string]string, client *http.Client) (courseInfo map[string]int) {
var err error
var res *http.Response
courseInfo = initCourseInfo(courseAndLesson)
for i := 0; i < 3; i++ {
d.mutexClient.RLock()
res, err = client.Get(DHUHostUrl + DHUSelectedUrl)
d.mutexClient.RUnlock()
if err == nil {
break
} else {
time.Sleep(time.Duration(i*2+1) * time.Second)
}
}
if err != nil {
return
}
doc, _ := goquery.NewDocumentFromResponse(res)
doc.Find("tr").Each(func(i int, s *goquery.Selection) {
lessonid := s.Find("td").Eq(0).Text()
_, err := strconv.Atoi(lessonid)
if err == nil {
l, ok := courseAndLesson[lessonid]
if ok {
courseInfo[l] = courseConflictStatus
}
}
})
return
}
func (v *ViPR) discover(cli *http.Client, url string) (map[string]string, error) {
user := env(v.Username, "VIPR_USERNAME", "Admin")
password := env(v.Password, "VIPR_PASSWORD", "ADMINISTRATOR")
request, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
request.SetBasicAuth(user, password)
resp, err := cli.Do(request)
if resp == nil || err != nil {
return nil, err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromResponse(resp)
if doc == nil || err != nil {
return nil, err
}
results := make(map[string]string)
// mac address is hidden in javascript and non changeable
doc.Find("td").Each(func(i int, s *goquery.Selection) {
p := strings.Fields(s.Text())
if len(p) > 0 && s.Next() != nil {
r := strings.Fields(strings.Replace(s.Next().Text(), "\"", "", -1))
if strings.HasPrefix(s.Text(), "Dataradio") {
results["version"] = strings.Join(p, " ")
}
if strings.HasPrefix(s.Text(), "MAC") {
if _, ok := results["macaddr"]; !ok && len(r) > 10 {
results["macaddr"] = strings.Join(r[5:10], ":")
}
}
if strings.HasPrefix(s.Text(), "RSSI") {
if len(r) > 1 {
results["signal"] = r[0]
}
}
switch strings.Join(p, " ") {
case "Modem Firmware Version":
results["firmware"] = strings.Replace(strings.Replace(strings.Join(r, " "), "(", "", -1), ")", "", -1)
case "Station Name":
results["name"] = strings.Join(r, " ")
case "Unit Status":
results["status"] = strings.Join(r, " ")
case "DC Input Voltage":
results["supply"] = strings.Join(r, " ")
case "Transceiver Temperature":
results["temperature"] = strings.Join(r, " ")
}
}
})
return results, nil
}
func (d *DHUStruct) getUserName(client *http.Client) (name string) {
var err error
var res *http.Response
for i := 0; i < 3; i++ {
d.mutexClient.RLock()
res, err = client.Get(DHUHostUrl + DHUSelfInfoUrl)
d.mutexClient.RUnlock()
if err == nil {
break
}
}
if err != nil {
return ""
}
dec := mahonia.NewDecoder("GB18030")
doc, _ := goquery.NewDocumentFromResponse(res)
doc.Find("tr").EachWithBreak(func(i int, s *goquery.Selection) bool {
info := s.Find("td").Eq(0).Text()
a := dec.ConvertString(info)
if a == "姓名" {
Newname := s.Find("td").Eq(1).Text()
name = dec.ConvertString(Newname)
return false
}
return true
})
return
}
func (h *Hongdian) discover(cli *http.Client, url string) (map[string]string, error) {
username := env(h.Username, "HONGDIAN_USERNAME", "admin")
password := env(h.Password, "HONGDIAN_PASSWORD", "admin")
request, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
request.SetBasicAuth(username, password)
resp, err := cli.Do(request)
if resp == nil || err != nil {
return nil, err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromResponse(resp)
if doc == nil || err != nil {
return nil, err
}
results := make(map[string]string)
doc.Find("div.setting input").Each(func(i int, s *goquery.Selection) {
k, _ := s.Attr("name")
v, _ := s.Attr("value")
if k == "" || v == "" {
return
}
results[k] = v
})
doc.Find("div.setting").Each(func(i int, s *goquery.Selection) {
p := strings.Fields(strings.Replace(s.Text(), ")(", ") ", -1))
if !(len(p) > 0) {
return
}
switch p[0] {
case "Capture(syspro.pattern)":
results["pattern"] = p[1]
case "Capture(syspro.num)":
results["num"] = p[1]
case "Capture(syspro.hard_release)":
results["hard"] = p[1]
case "Capture(syspro.soft_release)":
results["soft"] = p[1]
case "Capture(syslan.macaddr)":
results["macaddr"] = p[1]
case "Capture(share.ip_show)":
results["ip"] = p[1]
case "Capture(wireless.signal)":
results["sig"] = p[1]
}
})
return results, nil
}
// GetActionKey returns a key for actions (building, upgrading, ...)
func (c *Client) GetActionKey(resp *http.Response) (key string, found bool) {
doc, _ := goquery.NewDocumentFromResponse(resp)
keyInput := doc.Find("input[type=hidden][name=k]").First()
return keyInput.Attr("value")
}
// Vagrant scrapes from downloads page
func Vagrant() (string, string, error) {
res, err := http.Get("https://www.vagrantup.com/downloads.html")
if err != nil {
return "", "", err
}
defer res.Body.Close()
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
return "", "", err
}
var (
release string
url string
)
doc.Find("div.downloads a").Each(func(i int, s *goquery.Selection) {
link, _ := s.Attr("href")
if strings.HasSuffix(link, "_x86_64.rpm") {
r, _ := regexp.Compile("vagrant_(.*)_x86_64.rpm")
release = strings.TrimSpace(r.FindStringSubmatch(link)[1])
url = strings.TrimSpace(link)
}
})
if release != "" && url != "" {
return release, url, nil
}
return "", "", errors.New("unable to find release")
}
func (api *sisApi) Login(username string, password string) error {
// reset existing session
cookieJar, _ := cookiejar.New(nil)
api.client.Jar = cookieJar
api.loginPageDoc = nil
loginPath := api.baseUrl.Path
if api.baseUrl.RawQuery != "" {
loginPath += "?" + api.baseUrl.RawQuery
}
if loginPath == "" || loginPath == "/" { // login as patient if path is not set
loginPath = "/Login_Student_PXP.aspx"
}
loginUrl, err := api.baseUrl.Parse(loginPath)
if err != nil {
return err
}
// get login page and cookie
doc, err := api.getPage(loginPath)
if err != nil {
return err
}
// send login details
values := url.Values{}
doc.Find(`input[type="hidden"]`).Each(func(i int, s *goquery.Selection) {
values.Set(s.AttrOr("name", ""), s.AttrOr("value", ""))
})
values.Set("username", username)
values.Set("password", password)
req, err := http.NewRequest("POST", loginUrl.String(), strings.NewReader(values.Encode()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
res, err := api.client.Do(req)
if err != nil {
return err
}
doc, err = goquery.NewDocumentFromResponse(res)
if err != nil {
return err
}
if res.StatusCode == http.StatusOK {
userSelector := doc.Find(`.UserHead`)
if userSelector.Length() > 0 {
api.loginPageDoc = doc
return nil
} else {
html, _ := doc.Html()
return errors.New(fmt.Sprintf("unexpected page content %s", html))
}
} else {
return errors.New(fmt.Sprintf("unexpected http code %d", res.StatusCode))
}
return nil
}
func (utility *AuthUtility) Dispatch(session *Session) interface{} {
// Extract initial form values
resp, _ := session.Client.Get(LoginUrl)
doc, _ := goquery.NewDocumentFromResponse(resp)
formValues := url.Values{}
form := doc.Find("form#fm1").First()
form.Find("input").Each(func(i int, s *goquery.Selection) {
name, _ := s.Attr("name")
value, _ := s.Attr("value")
formValues.Set(name, value)
})
// Set credentials
formValues.Set("username", utility.Username)
formValues.Set("password", utility.Password)
// Attempt login
resp, _ = session.Client.PostForm(LoginUrl, formValues)
// Get token from cookies
var token string
parsedUrl, _ := url.Parse(LoginUrl)
cookies := session.Client.Jar.Cookies(parsedUrl)
for _, cookie := range cookies {
if cookie.Name == LoginCookie {
token = cookie.Value
}
}
return AuthResult{token, len(token) > 0}
}
func parseHTML(resp *http.Response) {
debug("extracting links")
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
debug("error creating goquery document")
return
}
defaultScheme := resp.Request.URL.Scheme
defaultAuthority := resp.Request.URL.Host
defaultPath := path.Dir(resp.Request.URL.EscapedPath()) + "/"
// use CSS selector found with the browser inspector
// for each, use index and item
doc.Find("body a").Each(func(index int, item *goquery.Selection) {
linkTag := item
link, _ := linkTag.Attr("href")
link = getFullLink(link, defaultScheme, defaultAuthority, defaultPath)
//debug(link)
if isScoped(link) {
req, err := http.NewRequest("GET", link, nil)
if err == nil {
select {
case reqFilterInputQ <- req:
//default:
}
}
}
})
}
