示例#1
0
func MakeRequest(url string) *browser.Browser {

	// Open a new browser needs to be a browser so run js
	bow := surf.NewBrowser()

	err := bow.Open(url)
	if err != nil {
		panic(err)
	}

	return bow
}
示例#2
0
文件: main.go 项目: kzuberi/nearlydue
func scrape(page string, p patron) (items []an_item) {
	bow := surf.NewBrowser()
	historyJar := jar.NewMemoryHistory()
	bow.SetHistoryJar(historyJar)

	// grab main page and click to login page
	err := bow.Open(page)
	if err != nil {
		panic(err)
	}

	fmt.Println(bow.Title())

	err = bow.Click("a:contains('My Account')")
	if err != nil {
		panic(err)
	}

	fmt.Println(bow.Title())

	// populate login form and submit
	form, err := bow.Form("form[action='/Mobile/MyAccount/Logon']")
	if err != nil {
		panic(err)
	}

	fmt.Println("logging in", p.Name)
	form.Input("barcodeOrUsername", p.Card)
	form.Input("password", p.Pin)

	fmt.Println(form)

	err = form.Submit()
	if err != nil {
		panic(err)
	}

	items_out := bow.Find("a[href='/Mobile/MyAccount/ItemsOut']")
	num_items_out, err := strconv.Atoi(strings.Split(items_out.Text(), " ")[0])

	if err != nil {
		panic(err)
	}

	if num_items_out > 0 {
		items = getItemsOut(bow)
		return items
	}

	return nil
}
示例#3
0
// сохраняет в файл csv результат запроса в файл с именем namef
func savehttptocsv(namef string, suri string, suri2 string) int {
	// Create a new browser and open reddit.
	bow := surf.NewBrowser()
	err := bow.Open(suri)
	if err != nil {
		panic(err)
	}
	err = bow.Open(suri2)
	if err != nil {
		panic(err)
	}
	rescsv := bow.Body()
	savestrtofile("report.csv", rescsv)
	return 0
}
示例#4
0
func scrape() (SolarOSReading, error) {
	bow := surf.NewBrowser()
	var reading SolarOSReading
	url, err := getScript()
	if err != nil {
		return SolarOSReading{}, err
	}
	err = bow.Open(url)
	if err != nil || bow.StatusCode() != 200 {
		return SolarOSReading{}, err
	}
	body := bow.Body()
	lifemeter := meter(body)
	if lifemeter != "" {
		reading.LifeMeter = lifemeter
	}
	reading.MoneySaved = dollarsSaved(body)
	reading.InstantaneousPower = instantMeter(body)
	reading.TreesSaved = treesSaved(body)
	reading.OilOffset = oilOffset(body)
	reading.CO2Offset = co2Offset(body)
	return reading, nil
}
示例#5
0
func getScript() (string, error) {
	// logs into SolarOS and scrapes the page, returning a usable URL
	bow := surf.NewBrowser()
	err := bow.Open("https://solaros.datareadings.com/")
	if err != nil || bow.StatusCode() != 200 {
		return "", err
	}
	fm := bow.Forms()[1]
	username := fm.Dom().Find("input").Nodes[0].Attr[3].Val
	password := fm.Dom().Find("input").Nodes[1].Attr[3].Val
	fm.Input(username, config.Username)
	fm.Input(password, config.Password)
	err = fm.Submit()
	if err != nil || bow.StatusCode() != 200 {
		return "", nil
	}
	cookie := fmt.Sprintf("%v", bow.SiteCookies()[0])
	cookie = strings.SplitN(cookie, "=", 2)[1]
	body := bow.Body()
	page := liftPage(body)
	watch := toWatch(body)
	url := dataURL(cookie, page, watch)
	return url, nil
}
示例#6
0
func main() {
	flag.Parse()
	spew.Config.Indent = "    "
	fmt.Printf("Version: %v\nUrl: %v\n", Version, Url)

	mech := surf.NewBrowser()
	err := mech.Open(Url)
	check(err)
	body := mech.Body()
	//doc, err := html.Parse(strings.NewReader(s))
	//doc, err := html.Parse(strings.NewReader(body))
	doc, err := gokogiri.ParseHtml([]byte(body))
	//doc, err := gokogiri.ParseHtml([]byte(body))
	check(err)

	div := doc.NodeById("middleContainer")
	//fmt.Printf("TypeOf div: %v\n", reflect.TypeOf(div))
	fmt.Printf("TypeOf div: %T\n", div)
	//spew.Printf("Div: %+v\n", div)
	//os.Exit(0)

	//content := string(div.Content())
	//fmt.Printf("Div: %+v\n", div.Content())
	//fmt.Println(content)
	//fmt.Println(div.String())

	if err := os.MkdirAll(*dir, os.ModePerm); err != nil {
		log.Fatal(err)
	}

	/*
			b := bytes.NewBufferString(string(doc.Data))
			//b := bytes.NewBufferString(string(body))
			//b := bytes.NewBufferString(string(div.String()))
			z := html.NewTokenizer(b)

			depth := 0
			for {
				tt := z.Next()

				fmt.Println("======================================================================")
				spew.Printf("z: %+v\n", z)
				fmt.Println("======================================================================")
				switch {
				case tt == html.ErrorToken:
					// End of the document, we're done
					return
		//			/*
		//				case tt == html.StartTagToken:
		//					t := z.Token()
		//
		//					isAnchor := t.Data == "a"
		//					if isAnchor {
		//						fmt.Println("We found a link!")
		//						spew.Printf("Data: %+v\n", t)
		//					}
		//
				case tt == html.TextToken:
					if depth > 0 {
						// emitBytes should copy the []byte it receives,
						// if it doesn't process it immediately.
						fmt.Println("*********")
						fmt.Println(z.Token())
						fmt.Println("*********")
					}
				case tt == html.StartTagToken, tt == html.EndTagToken:
					tn, _ := z.TagName()
					if len(tn) == 1 && tn[0] == 'a' {
						if tt == html.StartTagToken {
							depth++
						} else {
							depth--
						}
					}
				}
			}
	*/
	//Rows of interest have the form:
	// <a href="/1577/" title="2015-9-14">Advent</a><br>
}
示例#7
0
// NewBrowser is generator for Browser
func NewBrowser(email, password string) *Browser {
	bw := surf.NewBrowser()
	bw.SetUserAgent(userAgent)

	return &Browser{Browser: bw, Email: email, Password: password}
}