Example #1
0
func TestSubmittingAComment(t *testing.T) {

	// Given a server
	_, server, _, err := LoginUser(t)
	if err != nil {
		t.Fatal(err)
	}
	// When an authenticated client requests /item?id=1
	id := 1
	res, err := http.Get(fmt.Sprintf(server.URL+"/item?id=%d", id))
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()
	// It should respond with status 200

	if want, got := 200, res.StatusCode; want != got {
		t.Fatalf("status : want '%v' got '%v' ", want, got)
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	csrf, ok := doc.Find("input[name='comment_csrf']").First().Attr("value")
	if !ok {
		t.Fatalf("csrf value not found for comment form")
	}
	initialCommentNumber := doc.Find(".comment").Length()
	formValues := url.Values{
		"comment_content":   {"this is a new comment"},
		"comment_csrf":      {csrf},
		"comment_submit":    {"submit"},
		"comment_parent_id": {"0"},
		"comment_goto":      {fmt.Sprintf("/item?id=%d", id)},
		"comment_thread_id": {fmt.Sprintf("%d", id)},
	}
	// when an authenicated client submits a valid comment
	res, err = http.Post(server.URL+gonews.Route{}.Reply(), "application/x-www-form-urlencoded", strings.NewReader(formValues.Encode()))
	if err != nil {
		t.Fatal(err)
	}
	defer res.Body.Close()
	// It should respond with status 200
	if want, got := 200, res.StatusCode; want != got {
		t.Fatalf("Status : want '%v' got '%v' ", want, got)
	}
	// The number of comments on the story page should have increased by one
	doc, err = goquery.NewDocumentFromResponse(res)
	if err != nil {
		t.Fatal(err)
	}
	if want, got := initialCommentNumber+1, doc.Find(".comment").Length(); want != got {
		t.Fatalf(".comment length : want '%v' got '%v' ", want, got)
	}
}
Example #2
0
func main() {

	login()
	for _, v := range cookies {
		fmt.Println(v)
	}
	var bash_str string = "#!/bin/bash \n"
	baseMap := map[string]string{}
	doc, _ := goquery.NewDocumentFromResponse(getResultHtml(spider_base_url))
	doc.Find(".lesson-info-h2").Each(func(i int, contentSelection *goquery.Selection) {
		selection := contentSelection.Find("a")
		base_href, _ := selection.Attr("href")
		dir_name := selection.Text()
		bash_str += "mkdir \"" + dir_name + "\"\n"
		baseMap[dir_name] = base_href
		fmt.Println(dir_name, "-->", base_href)

	})
	downloadList := []DownloadBean{}

	for k, v := range baseMap {
		doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v))
		doc.Find(".lessonvideo-list dd h2").Each(func(i int, contentSelection *goquery.Selection) {
			selection := contentSelection.Find("a")
			href, _ := selection.Attr("href")
			filename := selection.Text()
			fmt.Println(k, "-->", filename, "-->", href)
			bean := DownloadBean{dirname: k, href: href, filename: filename}
			downloadList = append(downloadList, bean)
		})
	}

	for _, v := range downloadList {
		doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v.href))
		doc.Find("source").Each(func(i int, contentSelection *goquery.Selection) {
			download_url, _ := contentSelection.Attr("src")
			one_file := "wget " + download_url + "  -O \"./" + v.dirname + "/" + v.filename + ".mp4\"\n"
			bash_str += one_file
			fmt.Println(one_file)
		})
	}

	err := ioutil.WriteFile("./download.sh", []byte(bash_str), 0x777)
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("写入脚本完成")

}
Example #3
0
// GetResources retrieves resource info
func GetResources(resp *http.Response) Resources {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	resWrapperSelector := doc.Find("div#resWrap")

	lumber := resWrapperSelector.Find("td#l4")
	clay := resWrapperSelector.Find("td#l3")
	iron := resWrapperSelector.Find("td#l2")
	crop := resWrapperSelector.Find("td#l1")

	lumberProd, lumberStored, lumberCapa := parseResource(lumber)
	clayProd, clayStored, _ := parseResource(clay)
	ironProd, ironStored, _ := parseResource(iron)
	cropProd, cropStored, cropCapa := parseResource(crop)

	return Resources{
		Granary{
			Capacity: cropCapa,
			Crop:     cropStored,
		},
		Warehouse{
			Capacity: lumberCapa,
			Lumber:   lumberStored,
			Clay:     clayStored,
			Iron:     ironStored,
		},
		Production{
			lumberProd,
			clayProd,
			ironProd,
			cropProd,
		},
	}
}
Example #4
0
//解析函数定义,声明必须相同,下同
func Parser(httpRes *http.Response) ([]string, []basic.Item) {
	//两个需要返回的列表
	linklist := make([]string, 0)     //下一步需要请求的链接
	itemlist := make([]basic.Item, 0) //保存的数据类型为 map[string]interface{}

	//自定义部分
	//抓取所有链接
	doc, _ := goquery.NewDocumentFromResponse(httpRes)
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		link, exits := s.Attr("href")
		if exits {
			link = basic.CheckLink(link)
			if link != "" {
				linklist = append(linklist, link)
			}
		}
	})
	//保存每个页面的标题
	title := strings.TrimSpace(doc.Find("head title").Text())
	if title != "" {
		item := make(map[string]interface{})
		item["标题"] = title
		itemlist = append(itemlist, item)
	}

	return linklist, itemlist
}
Example #5
0
func (api *ExternalAPIEngine) DoRequest(
	arg APIArg, req *http.Request, restype int) (
	ar *ExternalAPIRes, hr *ExternalHTMLRes, tr *ExternalTextRes, err error) {

	var resp *http.Response
	var jw *jsonw.Wrapper

	resp, jw, err = doRequestShared(api, arg, req, (restype == XAPIResJSON))
	if err != nil {
		return
	}

	switch restype {
	case XAPIResJSON:
		ar = &ExternalAPIRes{resp.StatusCode, jw}
	case XAPIResHTML:
		var goq *goquery.Document
		goq, err = goquery.NewDocumentFromResponse(resp)
		if err == nil {
			hr = &ExternalHTMLRes{resp.StatusCode, goq}
		}
	case XAPIResText:
		var buf bytes.Buffer
		_, err := buf.ReadFrom(resp.Body)
		defer resp.Body.Close()
		if err == nil {
			tr = &ExternalTextRes{resp.StatusCode, string(buf.Bytes())}
		}
	default:
		err = fmt.Errorf("unknown restype to DoRequest")
	}
	return
}
Example #6
0
func harvest_board_indices(board_url string, board_name string) []BoardIndexPage {
	var ret []BoardIndexPage

	doc, err := goquery.NewDocumentFromResponse(ptt_get(board_url))
	if err != nil {
		log.Fatal(err)
	}

	re := regexp.MustCompile("/bbs/" + board_name + "/index([0-9]+)\\.html")
	doc.Find("a[href^='/bbs/" + board_name + "/index']").Each(func(_ int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if !exists {
			return
		}
		matched := re.FindStringSubmatch(href)
		if len(matched) == 0 {
			return
		}
		pn, err := strconv.Atoi(matched[1])
		if err != nil {
			log.Fatal(err)
		}
		ret = append(ret, BoardIndexPage{pn, href})
	})

	if ret[0].page_number > ret[1].page_number {
		ret[0], ret[1] = ret[1], ret[0]
	}

	for i := ret[0].page_number + 1; i < ret[1].page_number; i++ {
		ret = append(ret, BoardIndexPage{i, "/bbs/" + board_name + "/index" + strconv.Itoa(i) + ".html"})
	}

	return ret
}
Example #7
0
func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		fmt.Fprintln(os.Stderr, "yomikata [word]")
		os.Exit(1)
	}
	word := flag.Arg(0)
	resp, err := http.Get("http://yomikata.org/word/" + word)
	if err != nil {
		fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
		os.Exit(1)
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
		os.Exit(1)
	}
	if doc.Find("#word").Text() == "" {
		fmt.Fprintln(os.Stderr, "わかりません")
		os.Exit(1)
	}
	if *exact {
		fmt.Println(doc.Find(".spAns .psAns").First().Text())
	} else {
		doc.Find(".spAns").Each(func(_ int, s *goquery.Selection) {
			fmt.Printf("%s (%s)\n", s.Find(".psAns").Text(), s.Find(".psPt").Text())
		})
	}
}
Example #8
0
// GetStatistics retrieves statistics
func GetStatistics(resp *http.Response) Statistics {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	s := doc.Find("tr.hl").First()

	positionStr := strings.TrimRight(s.Find(".ra").Text(), ".")
	position, _ := strconv.Atoi(positionStr)

	username := strings.Trim(s.Find(".pla").Find("a").Text(), " ")
	alliance := s.Find("al").Text()

	popStr := s.Find(".pop").Text()
	pop, _ := strconv.Atoi(popStr)

	villageCountStr := s.Find(".vil").Text()
	villageCount, _ := strconv.Atoi(villageCountStr)

	stats := Statistics{
		position,
		username,
		alliance,
		pop,
		villageCount,
	}

	return stats
}
Example #9
0
// FetchRzrqSumData 抓取数据
func FetchCwzb(code, ctype string) ([]*Cwzb, error) {
	ctypeStr := "K"
	if ctype == "sz" {
		ctypeStr = "J"
	}
	formt := "http://quotes.cnfol.com/new/f10/cwzb/%s%s.html"

	resp, err := wget.Get(fmt.Sprintf(formt, code, ctypeStr))
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	doc.Find("table tr").Each(func(i int, s *goquery.Selection) {
		td := s.Find("td")
		if td.Length() < 5 {
			return
		}

		td.Each(func(i int, s *goquery.Selection) {
			if i == 0 {
				return
			}
			fmt.Println(s.Text())
		})
	})

	return nil, nil
}
Example #10
0
func mustParseResponse(resp *http.Response) *goquery.Document {
	gq, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}
	return gq
}
Example #11
0
func (a PlayStore) Search(sr SearchRequest) ([]App, error) {

	playurl, post := a.searchUrl(sr)

	body := strings.NewReader(post)
	req, err := http.NewRequest("POST", playurl, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")
	client := sr.Client
	resp, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)

	if err != nil {
		return nil, err
	}
	cards := doc.Find(".card")
	apps := make([]App, cards.Length())
	cards.Each(func(i int, s *goquery.Selection) {
		if val, ok := s.Attr("data-docid"); ok {
			apps[i].Package = val
		}
		if val, ok := s.Find(".title").Attr("title"); ok {
			apps[i].Name = val
		}

		if val, ok := s.Find(".subtitle").Attr("title"); ok {
			apps[i].PublisherName = val
		}

		if val, ok := s.Find(".card-click-target").Attr("href"); ok {
			apps[i].StoreUrl = "https://play.google.com" + val
		}

		if val, ok := s.Find(".cover-image").Attr("src"); ok {
			apps[i].IconUrl = val
		}

		if val, ok := s.Find(".current-rating").Attr("style"); ok {
			rating := strings.Split(val, " ")
			if len(rating) >= 2 {
				if rating_perc, err := strconv.ParseFloat(strings.TrimRight(rating[1], "%;"), 32); err == nil {
					apps[i].Rating = float32(rating_perc * 5.0 / 100.0)
				}

			}

		}

	})

	return apps, nil
}
Example #12
0
// Get the collection of the ProxyList by parse the http response
func (this *XiCiDaiLi) Result(response *http.Response) ([]contract.ProxyList, error) {
	doc, err := goquery.NewDocumentFromResponse(response)

	if err != nil {
		return nil, err
	}

	var proxyList []contract.ProxyList

	proxyTable := doc.Find("table#ip_list")
	proxyRows := proxyTable.Find("tr:nth-child(n+2)")

	proxyRows.Each(func(i int, s *goquery.Selection) {
		country := s.Find("td:nth-child(2) img").AttrOr("alt", "n/a")
		ip := s.Find("td:nth-child(3)").Text()
		port := s.Find("td:nth-child(4)").Text()
		protocol := s.Find("td:nth-child(7)").Text()

		if ip != "" && port != "" {
			proxyList = append(proxyList, contract.ProxyList{
				Ip:       ip,
				Port:     port,
				Protocol: protocol,
				Country:  strings.ToLower(country),
			})
		}
	})

	return proxyList, nil
}
func GetMajorList(client *http.Client) []string {
	banlist := map[string]string{"110431": "", "110432": "", "110511": "", "110521": "", "113110": "", "113111": ""}
	getmajorlist := []string{}
	res, err := client.Get(DHUHostUrl + DHUGetMajorURL)
	if err != nil {
		fmt.Println(err)
		fmt.Println("Something Wrong In GetMajorList")
	} else {
		doc, _ := goquery.NewDocumentFromResponse(res)
		doc.Find("select").Eq(1).Find("option").Each(func(i int, s *goquery.Selection) {
			sss, ok := s.Attr("value")
			if ok {
				_, testmap := banlist[sss]
				if testmap {
					return
				} else {
					getmajorlist = append(getmajorlist, sss)
				}
			} else {
				fmt.Println("Something Wrong")
			}
		})
		return getmajorlist
	}
	return nil
}
func GetAllLessons(res *http.Response) []CourseList {
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		fmt.Println(err)
		return nil
	} else {
		newlist := []CourseList{}
		dec := mahonia.NewDecoder("GB18030")
		doc.Find("tr").Each(func(i int, s *goquery.Selection) {
			selectid := s.Find("td").Eq(0).Text()
			_, err := strconv.Atoi(selectid)
			if err == nil {
				_, teachername, _ := dec.Translate([]byte(s.Find("td").Eq(6).Text()), true)
				newinfo := []CourseInfo{}
				s.Find("td").Eq(7).Find("tr").Each(func(i int, s *goquery.Selection) {
					_, weektime, _ := dec.Translate([]byte(s.Find("td").Eq(0).Text()), true)
					_, daytime, _ := dec.Translate([]byte(s.Find("td").Eq(1).Text()), true)
					info := CourseInfo{string(weektime), string(daytime)}
					newinfo = append(newinfo, info)
				})
				courselist := CourseList{selectid, string(teachername), newinfo}
				newlist = append(newlist, courselist)
			} else {
				return
			}
		})
		return newlist
	}
}
Example #15
0
func (this *Object) getHtml() (string, error) {

	res, err := request("GET", this.TargetUrl, "")
	if err != nil {
		return "", err
	}
	if res.StatusCode != 200 {
		return "", errors.New(fmt.Sprintf("http status not 200, it is %d", res.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return "", err
	}

	if err != nil {
		return "", err
	}
	html, err := this.parse(doc)
	if err != nil {
		return "", err
	}

	this.setArgs(doc)
	if err := this.setPageLimit(doc); err != nil {
		return "", err
	}

	return html, nil
}
Example #16
0
func (c Boot24Crawler) ParsePage(res *http.Response) *goquery.Document {
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		log.Fatal(err)
	}
	return doc
}
Example #17
0
// FetchStockData 抓取数据
func FetchStockData(code string) (*StockIndustry, *StockICB, error) {
	// http://stockData.stock.hexun.com/600028.shtml
	formt := "http://stockData.stock.hexun.com/%s.shtml"
	resp, err := wget.Get(fmt.Sprintf(formt, code))
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	tr := doc.Find("#list3 table.box6 tr")

	stockIndustry := &StockIndustry{}
	stockICB := &StockICB{}

	stockIndustry.Name, err = iconv.ConvertString(tr.Eq(7).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	stockICB.Name, err = iconv.ConvertString(tr.Eq(8).Find("td").Eq(1).Text(), "gb2312", "utf-8")
	if err != nil {
		return nil, nil, gos.DoError(err)
	}

	return stockIndustry, stockICB, nil
}
Example #18
0
func get_site(uri string, r *http.Request) ([]temp_item, error) {
	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	resp, err := client.Get(uri)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	var list_items []temp_item

	doc.Find("#bodyContent li").Not("#bodyContent #toc li").Each(func(i int, s *goquery.Selection) {
		listitem := s.Text()

		c.Infof("ITEM:\n" + listitem + "\n\n")

		q, err := filter_question(listitem)

		c.Infof("%v", err)

		if err == nil {
			list_items = append(list_items, q)
		}
	})

	//c.Infof("%+v\n", list_items)
	return list_items, nil
}
Example #19
0
func GetRecipeIds(categoryId int, page int) ([]int, error) {
	url := GetCategoryPageURL(categoryId, page)
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("failed to Get %s: %v", url, err)
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, fmt.Errorf("failed to parse response: %v", err)
	}

	recipeIds := []int{}
	doc.Find(".info").Each(func(i int, s *goquery.Selection) {
		s.Find(".name").Each(func(i int, s *goquery.Selection) {
			if href, ok := s.Find("a").Attr("href"); ok {
				id, err := strconv.Atoi(strings.Split(href, "/")[2])
				if err != nil {
					glog.Error("Failed to extract recipe id: %v", err)
				} else {
					recipeIds = append(recipeIds, id)
				}
			}
		})
	})
	return recipeIds, nil
}
Example #20
0
// BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
func BingDomain(domain, server string) (string, Results, error) {
	task := "bing"
	results := Results{}
	resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
	if err != nil {
		return task, results, err
	}
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return task, results, err
	}
	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
		u, err := url.Parse(s.Text())
		if err != nil || u.Host == "" {
			return
		}
		ip, err := LookupName(u.Host, server)
		if err != nil || ip == "" {
			cfqdn, err := LookupCname(u.Host, server)
			if err != nil || cfqdn == "" {
				return
			}
			ip, err = LookupName(cfqdn, server)
			if err != nil || ip == "" {
				return
			}

		}
		results = append(results, Result{Source: task, IP: ip, Hostname: u.Host})
	})
	return task, results, err
}
Example #21
0
func GetBuildings(resp *http.Response) Village {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	village := Village{}

	doc.Find("map#map2 area").EachWithBreak(func(index int, s *goquery.Selection) bool {
		parcelId := getIdFromURL(s.AttrOr("href", "fail"))

		name, level, empty := parseNameAndLevel(s.AttrOr("alt", "fail"))

		if empty {
			village.Parcels = append(village.Parcels, Parcel{
				Id:    parcelId,
				Empty: true,
			})
		} else {
			village.Parcels = append(village.Parcels, Parcel{
				Id:    parcelId,
				Empty: false,
				Building: Building{
					Name:  name,
					Level: level,
				},
			})
		}

		if parcelId == 40 { // 40 = Wall; it is repeated 3 times, this limits it to one
			return false
		} else {
			return true
		}
	})

	return village
}
Example #22
0
func (d *DHUStruct) ValidateStuCourseSelected(courseAndLesson map[string]string, client *http.Client) (courseInfo map[string]int) {
	var err error
	var res *http.Response
	courseInfo = initCourseInfo(courseAndLesson)
	for i := 0; i < 3; i++ {
		d.mutexClient.RLock()
		res, err = client.Get(DHUHostUrl + DHUSelectedUrl)
		d.mutexClient.RUnlock()
		if err == nil {
			break
		} else {
			time.Sleep(time.Duration(i*2+1) * time.Second)
		}
	}
	if err != nil {
		return
	}
	doc, _ := goquery.NewDocumentFromResponse(res)
	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
		lessonid := s.Find("td").Eq(0).Text()
		_, err := strconv.Atoi(lessonid)
		if err == nil {
			l, ok := courseAndLesson[lessonid]
			if ok {
				courseInfo[l] = courseConflictStatus
			}
		}
	})
	return
}
Example #23
0
File: vipr.go Project: ozym/dmc
func (v *ViPR) discover(cli *http.Client, url string) (map[string]string, error) {

	user := env(v.Username, "VIPR_USERNAME", "Admin")
	password := env(v.Password, "VIPR_PASSWORD", "ADMINISTRATOR")

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	request.SetBasicAuth(user, password)

	resp, err := cli.Do(request)
	if resp == nil || err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if doc == nil || err != nil {
		return nil, err
	}

	results := make(map[string]string)

	// mac address is hidden in javascript and non changeable
	doc.Find("td").Each(func(i int, s *goquery.Selection) {
		p := strings.Fields(s.Text())
		if len(p) > 0 && s.Next() != nil {
			r := strings.Fields(strings.Replace(s.Next().Text(), "\"", "", -1))
			if strings.HasPrefix(s.Text(), "Dataradio") {
				results["version"] = strings.Join(p, " ")
			}
			if strings.HasPrefix(s.Text(), "MAC") {
				if _, ok := results["macaddr"]; !ok && len(r) > 10 {
					results["macaddr"] = strings.Join(r[5:10], ":")
				}
			}
			if strings.HasPrefix(s.Text(), "RSSI") {
				if len(r) > 1 {
					results["signal"] = r[0]
				}
			}

			switch strings.Join(p, " ") {
			case "Modem Firmware Version":
				results["firmware"] = strings.Replace(strings.Replace(strings.Join(r, " "), "(", "", -1), ")", "", -1)
			case "Station Name":
				results["name"] = strings.Join(r, " ")
			case "Unit Status":
				results["status"] = strings.Join(r, " ")
			case "DC Input Voltage":
				results["supply"] = strings.Join(r, " ")
			case "Transceiver Temperature":
				results["temperature"] = strings.Join(r, " ")
			}
		}
	})

	return results, nil
}
Example #24
0
func (d *DHUStruct) getUserName(client *http.Client) (name string) {
	var err error
	var res *http.Response
	for i := 0; i < 3; i++ {
		d.mutexClient.RLock()
		res, err = client.Get(DHUHostUrl + DHUSelfInfoUrl)
		d.mutexClient.RUnlock()
		if err == nil {
			break
		}
	}
	if err != nil {
		return ""
	}
	dec := mahonia.NewDecoder("GB18030")
	doc, _ := goquery.NewDocumentFromResponse(res)
	doc.Find("tr").EachWithBreak(func(i int, s *goquery.Selection) bool {
		info := s.Find("td").Eq(0).Text()
		a := dec.ConvertString(info)
		if a == "姓名" {
			Newname := s.Find("td").Eq(1).Text()
			name = dec.ConvertString(Newname)
			return false
		}
		return true
	})
	return
}
Example #25
0
File: hongdian.go Project: ozym/dmc
func (h *Hongdian) discover(cli *http.Client, url string) (map[string]string, error) {
	username := env(h.Username, "HONGDIAN_USERNAME", "admin")
	password := env(h.Password, "HONGDIAN_PASSWORD", "admin")

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	request.SetBasicAuth(username, password)

	resp, err := cli.Do(request)
	if resp == nil || err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)
	if doc == nil || err != nil {
		return nil, err
	}

	results := make(map[string]string)

	doc.Find("div.setting input").Each(func(i int, s *goquery.Selection) {
		k, _ := s.Attr("name")
		v, _ := s.Attr("value")
		if k == "" || v == "" {
			return
		}
		results[k] = v

	})

	doc.Find("div.setting").Each(func(i int, s *goquery.Selection) {
		p := strings.Fields(strings.Replace(s.Text(), ")(", ") ", -1))
		if !(len(p) > 0) {
			return
		}

		switch p[0] {
		case "Capture(syspro.pattern)":
			results["pattern"] = p[1]
		case "Capture(syspro.num)":
			results["num"] = p[1]
		case "Capture(syspro.hard_release)":
			results["hard"] = p[1]
		case "Capture(syspro.soft_release)":
			results["soft"] = p[1]
		case "Capture(syslan.macaddr)":
			results["macaddr"] = p[1]
		case "Capture(share.ip_show)":
			results["ip"] = p[1]
		case "Capture(wireless.signal)":
			results["sig"] = p[1]
		}

	})

	return results, nil
}
Example #26
0
// GetActionKey returns a key for actions (building, upgrading, ...)
func (c *Client) GetActionKey(resp *http.Response) (key string, found bool) {
	doc, _ := goquery.NewDocumentFromResponse(resp)

	keyInput := doc.Find("input[type=hidden][name=k]").First()

	return keyInput.Attr("value")
}
Example #27
0
// Vagrant scrapes from downloads page
func Vagrant() (string, string, error) {
	res, err := http.Get("https://www.vagrantup.com/downloads.html")
	if err != nil {
		return "", "", err
	}
	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return "", "", err
	}

	var (
		release string
		url     string
	)
	doc.Find("div.downloads a").Each(func(i int, s *goquery.Selection) {
		link, _ := s.Attr("href")
		if strings.HasSuffix(link, "_x86_64.rpm") {
			r, _ := regexp.Compile("vagrant_(.*)_x86_64.rpm")
			release = strings.TrimSpace(r.FindStringSubmatch(link)[1])
			url = strings.TrimSpace(link)
		}
	})
	if release != "" && url != "" {
		return release, url, nil
	}
	return "", "", errors.New("unable to find release")
}
Example #28
0
func (api *sisApi) Login(username string, password string) error {
	// reset existing session
	cookieJar, _ := cookiejar.New(nil)
	api.client.Jar = cookieJar
	api.loginPageDoc = nil

	loginPath := api.baseUrl.Path
	if api.baseUrl.RawQuery != "" {
		loginPath += "?" + api.baseUrl.RawQuery
	}
	if loginPath == "" || loginPath == "/" { // login as patient if path is not set
		loginPath = "/Login_Student_PXP.aspx"
	}

	loginUrl, err := api.baseUrl.Parse(loginPath)
	if err != nil {
		return err
	}
	// get login page and cookie
	doc, err := api.getPage(loginPath)
	if err != nil {
		return err
	}

	// send login details
	values := url.Values{}
	doc.Find(`input[type="hidden"]`).Each(func(i int, s *goquery.Selection) {
		values.Set(s.AttrOr("name", ""), s.AttrOr("value", ""))
	})
	values.Set("username", username)
	values.Set("password", password)

	req, err := http.NewRequest("POST", loginUrl.String(), strings.NewReader(values.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	res, err := api.client.Do(req)
	if err != nil {
		return err
	}
	doc, err = goquery.NewDocumentFromResponse(res)
	if err != nil {
		return err
	}
	if res.StatusCode == http.StatusOK {
		userSelector := doc.Find(`.UserHead`)
		if userSelector.Length() > 0 {
			api.loginPageDoc = doc
			return nil
		} else {
			html, _ := doc.Html()
			return errors.New(fmt.Sprintf("unexpected page content %s", html))
		}
	} else {
		return errors.New(fmt.Sprintf("unexpected http code %d", res.StatusCode))
	}
	return nil
}
Example #29
0
func (utility *AuthUtility) Dispatch(session *Session) interface{} {
	// Extract initial form values
	resp, _ := session.Client.Get(LoginUrl)
	doc, _ := goquery.NewDocumentFromResponse(resp)

	formValues := url.Values{}

	form := doc.Find("form#fm1").First()
	form.Find("input").Each(func(i int, s *goquery.Selection) {
		name, _ := s.Attr("name")
		value, _ := s.Attr("value")

		formValues.Set(name, value)
	})

	// Set credentials
	formValues.Set("username", utility.Username)
	formValues.Set("password", utility.Password)

	// Attempt login
	resp, _ = session.Client.PostForm(LoginUrl, formValues)

	// Get token from cookies
	var token string

	parsedUrl, _ := url.Parse(LoginUrl)
	cookies := session.Client.Jar.Cookies(parsedUrl)
	for _, cookie := range cookies {
		if cookie.Name == LoginCookie {
			token = cookie.Value
		}
	}

	return AuthResult{token, len(token) > 0}
}
Example #30
0
func parseHTML(resp *http.Response) {
	debug("extracting links")

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		debug("error creating goquery document")
		return
	}
	defaultScheme := resp.Request.URL.Scheme
	defaultAuthority := resp.Request.URL.Host
	defaultPath := path.Dir(resp.Request.URL.EscapedPath()) + "/"

	// use CSS selector found with the browser inspector
	// for each, use index and item
	doc.Find("body a").Each(func(index int, item *goquery.Selection) {
		linkTag := item
		link, _ := linkTag.Attr("href")

		link = getFullLink(link, defaultScheme, defaultAuthority, defaultPath)
		//debug(link)
		if isScoped(link) {
			req, err := http.NewRequest("GET", link, nil)
			if err == nil {
				select {
				case reqFilterInputQ <- req:
					//default:
				}
			}
		}
	})
}