Beispiel #1
0
func getCookies() (cookies []webdriver.Cookie) {
	chromeDriver := webdriver.NewChromeDriver("/usr/local/bin/chromedriver")
	err := chromeDriver.Start()
	if err != nil {
		log.Println(err)
	}

	defer chromeDriver.Stop()

	desired := webdriver.Capabilities{"Platform": "OSX"}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		log.Println(err)
	}

	defer session.Delete()

	err = session.Url("https://lmet.aiwip.com/accounts/login/")

	if err != nil {
		log.Println(err)
	}

	passwordEl, err := session.FindElement(webdriver.CSS_Selector, "#id_password")
	if err != nil {
		panic(err)
	}

	passwordEl.SendKeys("")

	usernameEl, err := session.FindElement(webdriver.CSS_Selector, "#id_username")
	if err != nil {
		panic(err)
	}

	usernameEl.SendKeys("*****@*****.**")
	if err != nil {
		panic(err)
	}

	err = usernameEl.Submit()
	if err != nil {
		panic(err)
	}

	cookies, err = session.GetCookies()
	if err != nil {
		panic(err)
	}

	time.Sleep(5 * time.Second)
	fmt.Printf("Cookies: %+v", cookies)
	return cookies
}
Beispiel #2
0
func main() {
	chromeDriver := webdriver.NewChromeDriver("/usr/local/bin/chromedriver")
	err := chromeDriver.Start()
	if err != nil {
		log.Fatal(err)
	}
	defer chromeDriver.Stop()

	desired := webdriver.Capabilities{"loggingPrefs": logingPrefs}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		log.Fatal(err)
	}
	defer session.Delete()

	_, err = session.Log("performance")
	if err != nil {
		log.Fatal(err)
	}

	err = session.Url("https://www.raintank.io")
	if err != nil {
		log.Fatal(err)
	}

	logs, err := session.Log("performance")
	if err != nil {
		log.Fatal(err)
	}

	e, err := events.NewFromLogEntries(logs)
	if err != nil {
		log.Fatal(err)
	}

	har, err := httpArchive.CreateHARFromEvents(e)
	if err != nil {
		log.Fatal(err)
	}
	harJson, err := json.Marshal(har)
	eventJson, err := json.Marshal(e)

	// write out the HAR file and the RAW chromeEvens to file
	ioutil.WriteFile("./chromdriver.har", harJson, 0644)
	ioutil.WriteFile("./chromdriver.json", eventJson, 0644)
}
Beispiel #3
0
func main() {
	chromeDriver := webdriver.NewChromeDriver("/Users/zion/bin/chromedriver")
	err := chromeDriver.Start()
	if err != nil {
		log.Println(err)
	}
	desired := webdriver.Capabilities{"Platform": "Linux"}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		log.Println(err)
	}
	err = session.Url("http://www.baidu.com.cn")
	if err != nil {
		log.Println(err)
	}

	time.Sleep(10 * time.Second)
	session.Delete()
	chromeDriver.Stop()
}
Beispiel #4
0
func main() {
	flag.Parse()
	log.Print("start from", *start, " to ", *end)

	mgoSesssion, err := mgo.Dial("mongodb://localhost/shops")
	if err != nil {
		panic(err)
	}
	defer mgoSesssion.Close()

	chromeDriver := webdriver.NewChromeDriver(*pathToDriver)
	err = chromeDriver.Start()
	if err != nil {
		panic(err)
	}
	desired := webdriver.Capabilities{"Platform": *platform}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		panic(err)
	}

	for vendorID := *start; vendorID < *end; vendorID++ {
		err := SetVendorData(mgoSesssion, session, vendorID)
		if err != nil {
			log.Print("error for processed ", vendorID, err.Error())
		}
		if (vendorID % 1000) == 0 {
			log.Print("processed ", vendorID)
		}
	}

	if !*notCloseBrowser {
		session.Delete()
		chromeDriver.Stop()
	}
	log.Print("processed ", *end-*start)
}
func main() {
	chromeDriver := webdriver.NewChromeDriver("/usr/local/bin/chromedriver")

	err := chromeDriver.Start()
	if err != nil {
		panic(err)
	}
	defer chromeDriver.Stop()

	desired := webdriver.Capabilities{"Platform": "Mac"}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		panic(err)
	}

	session.Url("https://somewhere/someplace/awesomedata.html")
	source, err := session.Source()
	if err != nil {
		panic(err)
	}

	r := strings.NewReader(source)
	doc, _ := goquery.NewDocumentFromReader(r)
	doc.Find("tbody > tr").Each(func(i int, s *goquery.Selection) {
		name := s.Find("td:nth-child(3)")
		team := s.Find("td:nth-child(4)")
		time := s.Find("td:nth-child(5)")
		fmt.Printf("%s,%s,%s\n",
			strings.TrimSpace(name.Text()),
			strings.TrimSpace(team.Text()),
			strings.TrimSpace(time.Text()))
	})
	session.Delete()
	chromeDriver.Stop()
}
func GetGames(chromeDriverPath string) []matchBlock {

	//adding chromedriver and opening page
	chromeDriver := webdriver.NewChromeDriver(chromeDriverPath)
	err := chromeDriver.Start()
	if err != nil {
		log.Println(err)
	}
	desired := webdriver.Capabilities{"Platform": "Linux"}
	required := webdriver.Capabilities{}
	session, err := chromeDriver.NewSession(desired, required)
	if err != nil {
		log.Println(err)
	}
	err = session.Url(FLASHSCORE_URL)
	if err != nil {
		log.Println(err)
	}

	//wait for page to load
	time.Sleep(FLASHSCORE_LOAD_TIME * time.Second)

	//array game blocks
	var blocks []matchBlock

	//find all blocks with games
	HTMLblocks, _ := session.FindElements(webdriver.CSS_Selector, "table.soccer")
	for i := 0; i < len(HTMLblocks); i++ {

		var matches []match

		//finding country name
		element, err := HTMLblocks[i].FindElement(webdriver.ClassName, "country_part")
		if err != nil {
			fmt.Println("Can not find country")
		}
		//deleting ":" from country name
		textValue, err := element.Text()
		if err != nil {
			fmt.Println("Can not extract text")
		}
		countryName := strings.Replace(textValue, ":", "", -1)
		//finding league name
		element, err = HTMLblocks[i].FindElement(webdriver.ClassName, "tournament_part")
		leagueName, err := element.Text()
		if err != nil {
			fmt.Println("Can not extract league name text")
		}

		//finding all games in current league
		matchTRs, err := HTMLblocks[i].FindElements(webdriver.CSS_Selector, "tbody tr")
		if err != nil {
			fmt.Println("Can not find matches in league")
		}
		for j := 0; j < len(matchTRs); j++ {
			//finding team names
			homeTeamNameEl, err := matchTRs[j].FindElement(webdriver.CSS_Selector, "span.padr")
			if err != nil {
				fmt.Println("Can not find homeTeamName")
			}
			awayTeamNameEl, err := matchTRs[j].FindElement(webdriver.CSS_Selector, "span.padl")
			if err != nil {
				fmt.Println("Can not find awayTeamName")
			}
			homeTeamName, err := homeTeamNameEl.Text()
			if err != nil {
				fmt.Println("Can not extract homeTeamName text")
			}
			awayTeamName, err := awayTeamNameEl.Text()
			if err != nil {
				fmt.Println("Can not extract awayTeamName text")
			}
			//finding scores
			tdScoreEl, err := matchTRs[j].FindElement(webdriver.ClassName, "score")
			if err != nil {
				fmt.Println("Can not find score element")
			}
			tdScoreText, err := tdScoreEl.Text()
			if err != nil {
				fmt.Println("Can not extract text score")
			}
			//if game is playing or finished
			goalsHome := ""
			goalsAway := ""
			tdLength := len(tdScoreText)
			if tdLength == 5 {
				goalsHome = string(tdScoreText[0])
				goalsAway = string(tdScoreText[tdLength-1])
			}
			//finding start time
			tdTimeEl, err := matchTRs[j].FindElement(webdriver.ClassName, "time")
			if err != nil {
				fmt.Println("Can not time element")
			}
			startTime, err := tdTimeEl.Text()
			if err != nil {
				fmt.Println("Can not extract text from time")
			}
			matches = append(matches, match{homeTeamName: homeTeamName, awayTeamName: awayTeamName, goalsHome: goalsHome, goalsAway: goalsAway, startTime: startTime})
		}

		blocks = append(blocks, matchBlock{countryName: countryName, leagueName: leagueName, matches: matches})

	}

	session.Delete()
	chromeDriver.Stop()

	return blocks

}