Ejemplo n.º 1
0
func main() {
	var url = "http://toutiao.io"

	p, err := goquery.ParseUrl(url)

	if err != nil {
		panic(err)
	}
	// toutiao.io
	title := p.Find("title").Text()

	fmt.Println(title)

	t := p.Find(".title a")
	for i := 0; i < t.Length(); i++ {
		d := t.Eq(i).Text()
		l := t.Eq(i).Attr("href")
		c := p.Find(".summary a").Eq(i).Text()
		fmt.Println(l, d, "||", c)
	}

	// geek.csdn.net
	p, err = goquery.ParseUrl("http://geek.csdn.net/hotest")
	if err != nil {
		panic(err)
	}

	title = p.Find("title").Text()
	t = p.Find("a.title")

	fmt.Println(title)
	for i := 0; i < t.Length(); i++ {
		l := t.Eq(i).Attr("href")
		c := t.Eq(i).Text()
		fmt.Println(l, c)
	}
	// ituring.com
	p, err = goquery.ParseUrl("http://www.ituring.com.cn/")
	if err != nil {
		panic(err)
	}

	title = p.Find("title").Text()
	t = p.Find(".arc-list").Eq(0)
	x, _ := goquery.ParseString(t.Html())

	t = x.Find("dt a")

	fmt.Println(title)

	for i := 0; i < t.Length(); i++ {
		c := t.Eq(i).Text()
		l := t.Eq(i).Attr("href")
		fmt.Printf("http://www.ituring.com.cn/%s %s\n", l, c)
	}
}
func GetDownloadLinks(url string) {
	/*
	 *通过传入的书籍URL地址;提取到下载地址
	 *提取书籍的名称和url地址
	 *
	 */
	fmt.Println("\n")
	fmt.Println("书籍地址", url)
	r, err := goquery.ParseUrl(url)
	if err != nil {
		panic(err)
	} else {
		text := r.Find(".download-link")
		bookName := r.Find("h1").Text()
		fmt.Println("书籍名称:", bookName)
		//fmt.Println(text)
		for i := 0; i < text.Length(); i++ {
			downloadlink := text.Eq(i).Attr("href")
			fmt.Println("下载地址", downloadlink)
			DownloadBook(downloadlink, bookName)
		}

	}

}
Ejemplo n.º 3
0
func ParsePanMap(date string, history bool) {
	bet_url := strings.Replace(myinit.DateUrl, "TTT", date, -1)
	fmt.Println("bet_url:", bet_url)

	pan_url := myinit.PanUrl

	html_obj, _ := goquery.ParseUrl(bet_url)

	schedule_trs := html_obj.Find(".bet_table tbody tr")
	for i, _ := range schedule_trs {
		schedule_int_info := make(map[string]int)
		schedule_string_info := make(map[string]string)

		// insert schedule
		fid, _ := strconv.Atoi(schedule_trs.Eq(i).Attr("fid"))
		schedule_int_info["schedule_fenxi_id"] = int(fid)
		schedule_string_info["schedule_home"] = common.ConvToGB(schedule_trs.Eq(i).Attr("homesxname"))
		schedule_string_info["schedule_guest"] = common.ConvToGB(schedule_trs.Eq(i).Attr("awaysxname"))
		schedule_string_info["schedule_date"] = schedule_trs.Eq(i).Attr("pdate")

		schedule_pan_url := strings.Replace(pan_url, "TTT", strconv.Itoa(fid), -1)

		GetPanValue(schedule_pan_url, fid, schedule_string_info, date)

	}
}
Ejemplo n.º 4
0
Archivo: pan.go Proyecto: ryancsq/test
func checkPanUseable(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
	pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)

	odds_tr := pan_html_obj.Find(".table_cont table tbody tr")

	if odds_tr.Length() == 0 {
		return false
	}
	for i := 0; i < odds_tr.Length(); i++ {
		tr_item := odds_tr.Eq(i)
		td_of_company := tr_item.Find("td").Eq(1)
		if td_of_company.Find("p a").Attr("title") == "" {
			continue
		}

		is_useable := checkPanTr(tr_item, schedule_fenxi_id)
		if is_useable == false {
			return false
		}
	}
	if odds_tr.Length() >= 30 {
		ajax_res := checkPanUseableFromAjax(30, schedule_fenxi_id)
		if ajax_res == false {
			return false
		}
	}
	return true
}
Ejemplo n.º 5
0
func ParseBetUrl(date string, history bool) {
	bet_url := getBetUrl(date)
	html_obj, _ := goquery.ParseUrl(bet_url)
	schedule_trs := html_obj.Find(".bet_table tbody tr")
	for i, _ := range schedule_trs {
		is_end := schedule_trs.Eq(i).Attr("isend")
		if is_end == "1" && history == false {
			continue
		}

		today := common.GetToday()
		schedule_is_today := today == schedule_trs.Eq(i).Attr("pdate")
		//		fmt.Println("schedule_is_today:===",schedule_is_today)
		if schedule_is_today == false && history == false {
			//			continue
		}

		schedule_int_info, schedule_string_info := pareseScheduleTR(schedule_trs.Eq(i))
		schedule.Add(schedule_int_info, schedule_string_info)
		// end insert schedule

		//parse pan data
		res := ParsePanByScheduleFenxiId(schedule_int_info["schedule_fenxi_id"], date, schedule_string_info)
		if res == false {
			continue
		}
		//计算预测比率
		calcScheduleResult(schedule_int_info, schedule_string_info)
		//				return

	}
}
Ejemplo n.º 6
0
func GetZhihuQuestionList(url string) {
	/*
	 *  通过传入的url、分析提取url里面的问题列表
	 *  用for循环提交分页URL地址
	 *  并用goquery查找页面内容是否存在、用以判断是否还有分页
	 *  用提取到的短URL加上统一URL地址头得到某一个完整URL地址
	 */
	urlHeader := "http://www.zhihu.com"
	for i := 1; i < 100; i++ {
		url := url + strconv.Itoa(i)
		fmt.Println(url)
		r, err := goquery.ParseUrl(url)
		if err != nil {
			panic(err)
		} else {
			text := r.Find(".zm-item-title a") //查找所有问题列表
			if text.Length() > 0 {
				//取到text里面的所有"href"属性的数据
				for i := 0; i < text.Length(); i++ {
					singleUrl := urlHeader + text.Eq(i).Attr("href")
					urlList = append(urlList, singleUrl)
					GetSubjectBody(singleUrl)
				}
			} else { //如果text的长度小于0表示没有找到
				fmt.Print(".........没有文章了.......\n")
				break
			}
		}
	}

}
Ejemplo n.º 7
0
func ParseBetUrl(date string, history bool) {
	bet_url := getBetUrl(date)
	html_obj, _ := goquery.ParseUrl(bet_url)
	schedule_trs := html_obj.Find(".bet_table tbody tr")
	for i, _ := range schedule_trs {
		is_end := schedule_trs.Eq(i).Attr("isend")
		if is_end == "1" && history == false {
			continue
		}

		schedule_int_info, schedule_string_info := pareseScheduleTR(schedule_trs.Eq(i), date)
		//if(schedule_int_info["schedule_fenxi_id"]!=556793){
		//	continue
		//}
		//parse pan data
		res := ParsePanByScheduleFenxiId(schedule_int_info["schedule_fenxi_id"], date, schedule_string_info)
		if res == false {
			continue
		}
		schedule.Add(schedule_int_info, schedule_string_info)
		//计算预测比率
		calcScheduleResult(schedule_int_info, schedule_string_info)
		//				return

	}
}
func GetZi5PageUrl(category string) {
	/*
	 *通过传入的分类标签;得到分类地址
	 *用For循环传递分页页码;然后用goquery得到单个书籍的URL地址;
	 *如果分页里面得不到值就认为分页结束
	 *
	 *
	 */
	url := "http://book.zi5.me/archives/book-gentre/"
	//var category string
	for i := 1; i < 100; i++ {
		url := url + category + "/page/" + strconv.Itoa(i) //分页地址
		fmt.Println("\n")
		fmt.Println("分页地址", i, url)
		r, err := goquery.ParseUrl(url)
		if err != nil {
			panic(err)
		} else {
			//查找<class="thumb-holder" > 和<a
			text := r.Find(".thumb-holder a")
			//fmt.Println(text)
			if text.Length() > 0 {
				//取到text里面的所有"href"属性的数据
				for x := 0; x < text.Length(); x++ {
					bookUrl := text.Eq(x).Attr("href")
					GetDownloadLinks(bookUrl)
				}
			} else { //如果text的长度小于0表示没有找到
				fmt.Print("本分类下已经没有书籍\n")
				break
			}
		}
	}
}
Ejemplo n.º 9
0
func main() {
	x, err := goquery.ParseUrl("http://www.youtube.com/watch?v=ob_nh1WMMzU")
	if err != nil {
		panic(err)
	}
	x.Find("#eow-title").Print()
	fmt.Println("---")
	x, err = goquery.ParseUrl("http://thepiratebay.se/search/one%20day%202011/0/99/0")
	if err != nil {
		panic(err)
	}
	x.Find("a.detLink").Print()
	fmt.Println("---")
	for _, v := range x.Find("a.detLink").HtmlAll() {
		fmt.Println(v)
	}
}
Ejemplo n.º 10
0
func main() {

	pool := mongo.NewDialPool("localhost:27018", 1000)

	var i int = 1
	chs := make([]chan bool, 50)
	hasMore := true
	for ; i <= 50; i++ {
		ch := make(chan bool)

		chs[i-1] = ch

		go func(i int, ch chan bool) {
			conn, _ := pool.Get()
			db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
			coll := db.C("foods")
			data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
			if nil != err {
				log.Fatalln("response fail ,", err)
				ch <- false
				return
			}

			nodes := data.Find("div.lp_result_list")
			size := nodes.Find("li").Length()
			if size <= 0 {
				hasMore = false
			}
			for idx := 0; idx < size; idx++ {
				item := nodes.Find("li").Eq(idx)

				link := item.Find("a")
				href := link.Attr("href")
				name := link.Attr("title")
				img := item.Find("img").Attr("src")
				// log.Println(name, "|", href, "|", img)
				if len(name) > 0 {

					err := coll.Upsert(mongo.M{"name": name}, mongo.M{"name": name, "img_url": img, "link": href})

					// log.Println("insert mongo|", err, "|", href)
					log.Println("err", err, "name:", name, "link:", link, "href:", href, "img", img)
				}
			}
			ch <- true
			log.Println(i)

		}(i, ch)
	}

	for i, val := range chs {
		<-val
		log.Println("end:", i)
	}
}
Ejemplo n.º 11
0
func ParseResultUrl(date string, history bool) (res bool) {
	if date == "" {
		return false
	}
	result_url := strings.Replace(myinit.ResultUrl, "DDD", date, -1)
	fmt.Println(result_url)
	schedule_string_info := make(map[string]string)
	schedule_float_info := make(map[string]float32)

	pan_string_info := make(map[string]string)
	pan_float_info := make(map[string]float32)
	html_obj, _ := goquery.ParseUrl(result_url)
	schedule_trs := html_obj.Find(".ld_table tbody tr")
	for i, _ := range schedule_trs {
		if i == 0 {
			continue
		}
		tr := schedule_trs.Eq(i)
		tds := tr.Find("td")
		schedule_string_info["schedule_result_no"] = common.ConvToGB(tds.Eq(0).Html())

		schedule_string_info["schedule_score"] = common.ConvToGB(tds.Eq(6).Html())
		schedule_string_info["schedule_spf_result"] = common.ConvToGB(tds.Eq(11).Html())
		schedule_float_info["schedule_spf_odd"] = common.ConvToFloat32(tds.Eq(12).Text())
		schedule_string_info["schedule_rqspf_result"] = common.ConvToGB(tds.Eq(8).Html())
		schedule_float_info["schedule_rqspf_odd"] = common.ConvToFloat32(tds.Eq(9).Text())
		schedule_string_info["schedule_zjq_result"] = common.ConvToGB(tds.Eq(14).Html())
		schedule_float_info["schedule_zjq_odd"] = common.ConvToFloat32(tds.Eq(15).Text())
		schedule_string_info["schedule_bqc_result"] = common.ConvToGB(tds.Eq(17).Html())
		schedule_float_info["schedule_bqc_odd"] = common.ConvToFloat32(tds.Eq(18).Text())

		pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
		pan_string_info["schedule_score"] = schedule_string_info["schedule_score"]
		pan_string_info["schedule_spf_result"] = schedule_string_info["schedule_spf_result"]
		pan_string_info["schedule_rqspf_result"] = schedule_string_info["schedule_rqspf_result"]
		pan_string_info["schedule_zjq_result"] = schedule_string_info["schedule_zjq_result"]
		pan_string_info["schedule_bqc_result"] = schedule_string_info["schedule_bqc_result"]

		has := schedule.CheckExistsByResultNoAndDate(schedule_string_info["schedule_result_no"], date)
		fmt.Println("has:", has)
		if has == false {
			continue
		}
		schedule.UpdateScheduleResult(date, schedule_float_info, schedule_string_info)
		asiapan.UpdateAsiaPanResult(date, schedule_float_info, pan_string_info)
		asiapanlog.UpdateAsiaPanResult(date, pan_float_info, pan_string_info)

	}
	return true
}
Ejemplo n.º 12
0
func GetSubjectBody(url string) {
	/*
	 *  通过传入的单个问题url、分析提取url里面的问题标题和内容
	 *  用goquery查找页面提取里面title作为邮件的subject、用html()作为body
	 *
	 */
	//var url = "http://www.zhihu.com/question/24859069"
	p, error := goquery.ParseUrl(url)
	if error != nil {
		panic(error)
	}
	subject := p.Find("title").Text()
	body := p.Html()
	fmt.Print(subject)
	GetConf(subject, body)
}
Ejemplo n.º 13
0
func main() {

	pool := mongo.NewDialPool("localhost:27018", 1000)

	var i int = 1
	ch := make([]chan bool, 20)
	for ; i <= 20; i++ {
		ch[i-1] = make(chan bool)
		go func(i int, ch chan bool) {
			conn, _ := pool.Get()
			db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
			coll := db.C("foods")
			data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
			if nil != err {
				log.Fatalln("response fail ,", err)
				ch <- false
				return
			}

			nodes := data.Find("#page-" + strconv.Itoa(i))
			size := nodes.Find("div.media").Length()
			for idx := 0; idx < size; idx++ {
				item := nodes.Find("div.media").Eq(idx)
				h2 := item.Find("h2")
				link := h2.Find("a")
				href := link.Attr("href")
				name := link.Attr("title")

				img := item.Find("img").Attr("src")
				// log.Println(name, "|", href, "|", img)
				if len(name) > 0 {
					err := coll.Insert(mongo.M{"name": name, "img_url": img, "link": href})
					log.Println("insert mongo|", err, "|", href)
				}
			}
			ch <- true
			log.Println(i)

		}(i, ch[i-1])
	}

	for i, val := range ch {
		<-val
		log.Println("end:", i)
	}
}
Ejemplo n.º 14
0
Archivo: pan.go Proyecto: ryancsq/test
func doParsePanUrl(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
	pan_int_info := make(map[string]int)
	pan_float_info := make(map[string]float32)
	pan_string_info := make(map[string]string)

	pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)

	schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
	home_td := schedule_item.Eq(0)
	guest_td := schedule_item.Eq(4)
	center_td := schedule_item.Eq(2)

	pan_int_info["schedule_fenxi_id"] = schedule_fenxi_id
	pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
	pan_string_info["schedule_no"] = schedule_string_info["schedule_no"]
	pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
	pan_string_info["schedule_league"] = schedule_string_info["schedule_league"]

	pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())
	pan_string_info["schedule_guest"] = common.ConvToGB(guest_td.Find("ul li a").Text())
	pan_string_info["schedule_game_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
	pan_string_info["schedule_date_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .game_time ").Text())

	odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
	for i := 0; i < odds_tr.Length(); i++ {
		tr_item := odds_tr.Eq(i)
		td_of_company := tr_item.Find("td").Eq(1)
		if td_of_company.Find("p a").Attr("title") == "" {
			continue
		}

		parse_res := doParsePanTr(tr_item, schedule_fenxi_id, pan_int_info, pan_float_info, pan_string_info, false)
		if parse_res == false {
			return false
		}
	}
	if odds_tr.Length() >= 30 {
		ajax_res := ParsePanUrlFromAjax(30, schedule_fenxi_id, pan_string_info)
		if ajax_res == false {
			return false
		}
	}

	return true
}
Ejemplo n.º 15
0
// use Goquery to fetch product-name from html
func fetchProductName(url string) {
	p, err := goquery.ParseUrl(url)
	if err != nil {
		panic(err)
	} else {
		pTitle := p.Find("title").Text() // fetch the content of title
		fmt.Println(pTitle)

		productList := p.Find(".product-name")
		priceList := p.Find(".product_price")
		for i := 0; i < productList.Length(); i++ {
			product := productList.Eq(i).Text()
			price := priceList.Eq(i).Text()
			fmt.Println(product + ": " + price)

		}

	}
}
Ejemplo n.º 16
0
func GetQuery(getUrl string) string {

	x, err := goquery.ParseUrl(getUrl)
	if err != nil {
		panic(err)
		return "ERROR! IN Get Query."
	}
	//x.Find(".info").Print()
	detail := x.Find(".content").Html()

	//格式化处理
	//移除HTML
	r := regexp.MustCompile("<.*?>")
	detail = r.ReplaceAllString(detail, " ")
	//移除换行
	r = regexp.MustCompile("[ \f\n\r\t\v]")
	detail = r.ReplaceAllString(detail, "")

	return detail
}
Ejemplo n.º 17
0
func getHmtl(url string) (string, error) {
	p, err := goquery.ParseUrl(url)
	if err != nil {
		return "", err
	} else {
		t := p.Find(".lyric-action")
		if t.Length() > 0 {
			for i := 0; i < 1; i++ { //只返回一条记录
				if strings.Contains(t.Html(), "down-lrc-btn") {
					lrc := strings.Split(strings.SplitN(t.Html(), "down-lrc-btn", 2)[1], "&#39;")
					return lrc[3], nil

				}

			}
		}

	}
	return "", errors.New("没能找到你需要的歌词")
}
Ejemplo n.º 18
0
func GetMusicID(name string) []interface{} {
	p, err := goquery.ParseUrl(url_music + name + url_music_break)
	if err != nil {
		fmt.Println(err)
	} else {
		var items []interface{}
		t := p.Find(".song-item-hook")
		if t.Length() > 0 {
			for i := 0; i < t.Length(); i++ {
				if strings.Contains(t.Eq(i).Html(), "data-songdata") {
					str := strings.SplitN(t.Eq(i).Html(), "data-songdata", 2)[0]
					str = strings.SplitN(str, "href=", 2)[1]
					if strings.Contains(str, " ") {
						str = strings.Split(str, " ")[0]
					} else {
						continue
					}
					if strings.Contains(str, "/") {
						item_ht := strings.Split(str, "/")
						str = item_ht[len(item_ht)-1]
					} else {
						continue
					}
					if strings.Contains(str, "\"") {
						str = strings.Split(str, "\"")[0]
					} else {
						continue
					}
					items = append(items, str)
				}
			}
		}
		return items

	}
	return nil
}
Ejemplo n.º 19
0
Archivo: pan.go Proyecto: ryancsq/test
func ParsePanUrl(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
	pan_int_info := make(map[string]int)
	pan_float_info := make(map[string]float32)
	pan_string_info := make(map[string]string)

	pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)

	schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
	home_td := schedule_item.Eq(0)
	guest_td := schedule_item.Eq(4)
	center_td := schedule_item.Eq(2)

	pan_int_info["schedule_fenxi_id"] = schedule_fenxi_id
	pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
	pan_string_info["schedule_no"] = schedule_string_info["schedule_no"]
	pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
	pan_string_info["schedule_league"] = schedule_string_info["schedule_league"]

	pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())
	pan_string_info["schedule_guest"] = common.ConvToGB(guest_td.Find("ul li a").Text())
	pan_string_info["schedule_game_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
	pan_string_info["schedule_date_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .game_time ").Text())

	odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
	for i := 0; i < odds_tr.Length(); i++ {
		tr_item := odds_tr.Eq(i)

		td_of_company := tr_item.Find("td").Eq(1)
		if td_of_company.Find("p a").Attr("title") == "" {
			continue
		}

		company_id := tr_item.Attr("id")
		pan_string_info["company_id"] = company_id
		pan_string_info["company_name"] = common.ConvToGB(td_of_company.Find("p a").Attr("title"))

		var is_big_company = 0
		if td_of_company.Find("p img").Attr("src") == "" {
			is_big_company = 0
		} else {
			is_big_company = 1
			fmt.Println("src:" + td_of_company.Find("p img").Attr("src"))
		}
		pan_int_info["is_big_company"] = is_big_company

		table_of_pan_detail := tr_item.Find("td .pl_table_data")

		table_of_opentime_pan := table_of_pan_detail.Eq(1)
		tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")
		open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
		pan_float_info["open_pan"] = float32(open_pan_32)
		pan_string_info["open_pan_desc"] = common.ConvToGB(tds_of_opentime_pan_table.Eq(1).Text())

		open_home_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(0).Text(), 32)
		open_guest_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(2).Text(), 32)
		pan_float_info["open_home_water"] = float32(open_home_water_32)
		pan_float_info["open_guest_water"] = float32(open_guest_water_32)

		td_of_pan_time := tr_item.Find("td time")

		pan_string_info["open_pan_time"] = td_of_pan_time.Eq(1).Text()

		table_of_realtime_pan := table_of_pan_detail.Eq(0)
		tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")

		real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
		pan_float_info["real_pan"] = float32(real_pan_32)
		pan_string_info["real_pan_desc"] = common.ConvToGB(tds_of_realtime_pan_table.Eq(1).Text())

		home_real_water_string := common.ConvToGB(tds_of_realtime_pan_table.Eq(0).Text())
		home_real_water_str := strings.Replace(home_real_water_string, "↑", "", -1)
		home_real_water_str = strings.Replace(home_real_water_str, "↓", "", -1)

		guest_real_water_string := common.ConvToGB(tds_of_realtime_pan_table.Eq(2).Text())
		guest_real_water_str := strings.Replace(guest_real_water_string, "↑", "", -1)
		guest_real_water_str = strings.Replace(guest_real_water_str, "↓", "", -1)

		home_real_water_32, _ := strconv.ParseFloat(home_real_water_str, 32)
		guest_real_water_32, _ := strconv.ParseFloat(guest_real_water_str, 32)

		pan_float_info["real_home_water"] = float32(home_real_water_32)
		pan_float_info["real_guest_water"] = float32(guest_real_water_32)
		fmt.Println("before home_real_water:", pan_float_info["real_home_water"])

		pan_string_info["pan_change_time"] = td_of_pan_time.Eq(0).Text()

		td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
		home_pan_change_type := common.ConvToGB(td_item_of_real_pan.Find("font").Text())
		home_pan_change_type = strings.TrimSpace(home_pan_change_type)
		pan_int_info["home_pan_change_type"] = 0

		if home_pan_change_type == "升" {
			pan_int_info["home_pan_change_type"] = 1
			pan_string_info["home_pan_change_type_desc"] = home_pan_change_type

		}
		if home_pan_change_type == "降" {
			pan_int_info["home_pan_change_type"] = -1
			pan_string_info["home_pan_change_type_desc"] = home_pan_change_type

		}

		home_water_up_down_flag := tds_of_realtime_pan_table.Eq(0).Attr("class")
		pan_int_info["home_water_change_type"] = 0
		if home_water_up_down_flag == "ping" {
			pan_int_info["home_water_change_type"] = -1            // down
			pan_string_info["home_water_change_type_desc"] = "水位降" // down
		}
		if home_water_up_down_flag == "ying" {
			pan_int_info["home_water_change_type"] = 1             // up
			pan_string_info["home_water_change_type_desc"] = "水位升" // up
		}
		fmt.Println("before2 home_real_water:", pan_float_info["real_home_water"])

		real_pan_string := strings.Replace(pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"], "", -1)
		real_pan_desc := strings.TrimSpace(real_pan_string)

		//		panmap.Add(pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
		//		panmap.Add(real_pan_desc, pan_float_info["real_pan"])

		fmt.Println("date:", pan_string_info["schedule_date"], pan_string_info["schedule_home"])
		fmt.Println("open:", pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
		fmt.Println("real:", real_pan_desc, pan_float_info["real_pan"])
		parse_change_data := ParsePanChangeUrl(schedule_fenxi_id, company_id, pan_int_info, pan_float_info, pan_string_info)
		if parse_change_data == false {
			continue
		}
		fmt.Println("before3 home_real_water:", pan_float_info["real_home_water"])

		if pan_float_info["open_pan"] > 0 || pan_float_info["real_pan"] > 0 {
			delete_asiapan := new(myinit.AsiaPan)
			del_result, _ := myinit.Engine.Where("schedule_fenxi_id=? ", schedule_fenxi_id).Delete(delete_asiapan)
			delete_schedule := new(myinit.Schedule)
			del_schedule_result, _ := myinit.Engine.Where("schedule_fenxi_id=? ", schedule_fenxi_id).Delete(delete_schedule)
			fmt.Println(del_result, del_schedule_result)
			fmt.Println("开盘>0 或者即时盘 >0")
			return false
		}

		predict1_result, predict1_cmt := analyse.AnalysePanResult1(pan_int_info, pan_float_info, pan_string_info)
		pan_string_info["predict1_result"] = predict1_result
		pan_string_info["predict1_cmt"] = predict1_cmt
		predict2_result, predict2_cmt := analyse.AnalysePanResult2(pan_int_info, pan_float_info, pan_string_info)
		pan_string_info["predict2_result"] = predict2_result
		pan_string_info["predict2_cmt"] = predict2_cmt

		// fmt.Println("company:" + company)
		// fmt.Println("home_pan_change_type:" + home_pan_change_type)
		// fmt.Println("is big company:" + is_big_company)
		// fmt.Println("change_time:" + change_time)
		// fmt.Println("open_time:" + open_time)
		// fmt.Println("flag:" + home_water_change_type + " " + home_water_up_down_flag)
		fmt.Println("home_real_water:", pan_float_info["real_home_water"])
		fmt.Println(home_real_water_32)
		fmt.Println(float32(home_real_water_32))
		fmt.Println("home_real_water water sting:" + home_real_water_str + home_real_water_string)
		fmt.Println("guest_real_water:", pan_float_info["real_guest_water"])
		fmt.Println("guest_real_water water sting:" + guest_real_water_str + guest_real_water_string)
		fmt.Println("pan:", pan_float_info["real_pan"], " ", real_pan_desc)

		// fmt.Println("open_home_water water:", open_home_water)
		// fmt.Println("open_guest_water water:", open_guest_water)
		// fmt.Println("open pan:", open_pan, " ", open_pan_desc)

		exist_asiapan := new(myinit.AsiaPan)
		has, _ := myinit.Engine.Where("schedule_fenxi_id=? AND company_id=? ", schedule_fenxi_id, company_id).Get(exist_asiapan)
		if has {
			fmt.Println(pan_string_info["company_name"] + "pan已存在!")
			if exist_asiapan.PanChangeTime != pan_string_info["pan_change_time"] {
				fmt.Println(pan_string_info["company_name"] + "pan有变化!")
				asiapan.UpdateAsiaPanInfo(pan_int_info, pan_float_info, pan_string_info)
			}
		} else {
			asiapan.Add(pan_int_info, pan_float_info, pan_string_info)
		}

		//		count_open_water := open_home_water +open_guest_water
		count_real_water := pan_float_info["real_home_water"] + pan_float_info["real_guest_water"]
		if count_real_water < 1.75 || count_real_water > 2 {
			//			fmt.Println("+++++++")
			//			fmt.Println(pan_float_info["real_home_water"])
			//			fmt.Println(pan_float_info["real_guest_water"])

			//			delete_asiapan2 := new(myinit.AsiaPan)
			//			delete2, _ := myinit.Engine.Where("schedule_fenxi_id=? AND company_id=? ", schedule_fenxi_id, company_id).Delete(delete_asiapan2)
			//			fmt.Println(delete2)
		}
	}
	if odds_tr.Length() >= 30 {
		ParsePanUrlFromAjax(30, schedule_fenxi_id, pan_string_info)

	}

	return true
}
Ejemplo n.º 20
0
func ParseBetUrl(date string, history bool) {
	bet_url := strings.Replace(myinit.DateUrl, "TTT", date, -1)
	if date == "" {
		bet_url = myinit.IndexUrl
	}
	fmt.Println("bet_url:", bet_url)

	pan_url := myinit.PanUrl

	html_obj, _ := goquery.ParseUrl(bet_url)
	//		fmt.Println(html_obj.HtmlAll())

	schedule_trs := html_obj.Find(".bet_table tbody tr")
	for i, _ := range schedule_trs {
		is_end := schedule_trs.Eq(i).Attr("isend")
		if is_end == "1" && history == false {
			//			fmt.Println("is_end")
			continue
		}

		schedule_int_info := make(map[string]int)
		schedule_string_info := make(map[string]string)

		// insert schedule
		fid, _ := strconv.Atoi(schedule_trs.Eq(i).Attr("fid"))
		schedule_int_info["schedule_fenxi_id"] = int(fid)
		schedule_string_info["schedule_home"] = common.ConvToGB(schedule_trs.Eq(i).Attr("homesxname"))
		schedule_string_info["schedule_guest"] = common.ConvToGB(schedule_trs.Eq(i).Attr("awaysxname"))
		schedule_string_info["schedule_date"] = schedule_trs.Eq(i).Attr("pdate")
		schedule_string_info["schedule_league"] = common.ConvToGB(schedule_trs.Eq(i).Attr("lg"))
		schedule_string_info["schedule_week_day"] = common.ConvToGB(schedule_trs.Eq(i).Attr("gdate"))
		schedule_string_info["schedule_no"] = schedule_trs.Eq(i).Attr("pname")
		schedule_string_info["schedule_rq_num"] = schedule_trs.Eq(i).Attr("rq")
		week_date := schedule_string_info["schedule_no"][0:1]
		schedule_string_info["schedule_result_no"] = myinit.WeekDesc[week_date] + schedule_string_info["schedule_no"][1:]

		schedule_string_info["schedule_bet_end_time"] = schedule_trs.Eq(i).Attr("pendtime")

		today := time.Now().Format("2006-01-02")
		schedule_is_today := today == schedule_string_info["schedule_date"]
		//		fmt.Println("schedule_is_today:===",schedule_is_today)
		if schedule_is_today == false && history == false {
			continue
		}
		schedule.Add(schedule_int_info, schedule_string_info)
		// end insert schedule

		schedule_pan_url := strings.Replace(pan_url, "TTT", strconv.Itoa(fid), -1)
		//		fmt.Println("schedule_pan_url:",schedule_pan_url)
		res := ParsePanUrl(schedule_pan_url, fid, schedule_string_info, date)
		fmt.Println(res)
		fmt.Println("---------")
		fmt.Println(schedule_string_info["schedule_date"])
		fmt.Println(schedule_string_info["schedule_no"])
		fmt.Println("---------")
		if res == false {
			continue
		}
		calcScheduleResult(schedule_int_info, schedule_string_info)
		return
	}
}
Ejemplo n.º 21
0
Archivo: pan.go Proyecto: ryancsq/test
func GetPanValue(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
	pan_int_info := make(map[string]int)
	pan_float_info := make(map[string]float32)
	pan_string_info := make(map[string]string)
	pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)
	schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
	home_td := schedule_item.Eq(0)
	pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
	pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())

	odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
	for i := 0; i < odds_tr.Length(); i++ {
		tr_item := odds_tr.Eq(i)
		td_of_company := tr_item.Find("td").Eq(1)
		if td_of_company.Find("p a").Attr("title") == "" {
			continue
		}
		pan_string_info["company_name"] = common.ConvToGB(td_of_company.Find("p a").Attr("title"))
		table_of_pan_detail := tr_item.Find("td .pl_table_data")
		table_of_opentime_pan := table_of_pan_detail.Eq(1)
		tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")
		open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
		pan_float_info["open_pan"] = float32(open_pan_32)
		pan_string_info["open_pan_desc"] = common.ConvToGB(tds_of_opentime_pan_table.Eq(1).Text())

		table_of_realtime_pan := table_of_pan_detail.Eq(0)
		tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")

		real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
		pan_float_info["real_pan"] = float32(real_pan_32)
		pan_string_info["real_pan_desc"] = common.ConvToGB(tds_of_realtime_pan_table.Eq(1).Text())

		td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
		home_pan_change_type := common.ConvToGB(td_item_of_real_pan.Find("font").Text())
		home_pan_change_type = strings.TrimSpace(home_pan_change_type)
		pan_int_info["home_pan_change_type"] = 0

		if home_pan_change_type == "升" {
			pan_int_info["home_pan_change_type"] = 1
			pan_string_info["home_pan_change_type_desc"] = home_pan_change_type

		}
		if home_pan_change_type == "降" {
			pan_int_info["home_pan_change_type"] = -1
			pan_string_info["home_pan_change_type_desc"] = home_pan_change_type

		}

		real_pan_string := strings.Replace(pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"], "", -1)
		real_pan_desc := strings.TrimSpace(real_pan_string)
		fmt.Println("====here====")
		fmt.Println(i)

		fmt.Println("date:", pan_string_info["schedule_date"], pan_string_info["schedule_home"], pan_string_info["company_name"])
		//		fmt.Println("open:", pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
		//		fmt.Println("real:", real_pan_desc, pan_float_info["real_pan"])
		//		fmt.Println("real desc:", pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"])
		fmt.Println("++end here+++")
		panmap.Add(pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
		panmap.Add(real_pan_desc, pan_float_info["real_pan"])
	}

	if odds_tr.Length() >= 30 {
		FetchTrByAjax(30, schedule_fenxi_id, pan_string_info["schedule_date"], pan_string_info["schedule_home"])
	}
	return true
}
Ejemplo n.º 22
0
Archivo: xorm.go Proyecto: ryancsq/test
func ParseOddUrl(schedule_odds_url string, schedulefid string, pname string, schedule_date_orig string) (a string) {
	odds_html, _ := goquery.ParseUrl(schedule_odds_url)

	schedule_item := odds_html.Find(".odds_hd_cont table tbody tr td")
	home_td := schedule_item.Eq(0)
	guest_td := schedule_item.Eq(4)
	center_td := schedule_item.Eq(2)

	schedule_home_name := convToGB(home_td.Find("ul li a").Text())
	schedule_guest_name := convToGB(guest_td.Find("ul li a").Text())
	schedule_game_desc := convToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
	//	schedule_date := convToGB(center_td.Find(".odds_hd_center .game_time ").Text())

	schedule_date := schedule_date_orig

	odds_tr := odds_html.Find(".table_cont table tbody tr")
	for i := 0; i < odds_tr.Length(); i++ {
		tr_item := odds_tr.Eq(i)

		td_of_company := tr_item.Find("td").Eq(1)

		company := convToGB(td_of_company.Find("p a").Attr("title"))
		if td_of_company.Find("p a").Attr("title") == "" {
			continue
		}

		var is_big_company = "0"
		if td_of_company.Find("p img").Attr("src") == "" {
			is_big_company = "0"
		} else {
			is_big_company = "1"
			fmt.Println("src:" + td_of_company.Find("p img").Attr("src"))
		}

		cid := tr_item.Attr("id")

		td_of_pan_time := tr_item.Find("td time")

		change_time := td_of_pan_time.Eq(0).Text()
		open_time := td_of_pan_time.Eq(1).Text()

		table_of_pan_detail := tr_item.Find("td .pl_table_data")
		table_of_realtime_pan := table_of_pan_detail.Eq(0)
		tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")

		home_water_up_down_flag := tds_of_realtime_pan_table.Eq(0).Attr("class")
		var home_water_change_type = "water_unknown"
		if home_water_up_down_flag == "ping" {
			home_water_change_type = "water_down" // down
		}
		if home_water_up_down_flag == "ying" {
			home_water_change_type = "water_up" // up
		}

		home_real_water_string := convToGB(tds_of_realtime_pan_table.Eq(0).Text())
		guest_real_water_string := convToGB(tds_of_realtime_pan_table.Eq(2).Text())

		real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
		real_pan := float32(real_pan_32)

		real_pan_desc := convToGB(tds_of_realtime_pan_table.Eq(1).Text())

		td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
		home_pan_change_type := convToGB(td_item_of_real_pan.Find("font").Text())
		home_pan_change_type = strings.TrimSpace(home_pan_change_type)
		home_real_water_str := strings.Replace(home_real_water_string, "↑", "", -1)
		home_real_water_str = strings.Replace(home_real_water_str, "↓", "", -1)

		guest_real_water_str := strings.Replace(guest_real_water_string, "↑", "", -1)
		guest_real_water_str = strings.Replace(guest_real_water_str, "↓", "", -1)

		table_of_opentime_pan := table_of_pan_detail.Eq(1)
		tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")

		open_home_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(0).Text(), 32)
		open_guest_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(2).Text(), 32)

		home_real_water_32, _ := strconv.ParseFloat(home_real_water_str, 32)
		guest_real_water_32, _ := strconv.ParseFloat(guest_real_water_str, 32)

		open_home_water := float32(open_home_water_32)
		open_guest_water := float32(open_guest_water_32)

		home_real_water := float32(home_real_water_32)
		guest_real_water := float32(guest_real_water_32)

		open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
		open_pan := float32(open_pan_32)

		if open_pan > 0 || real_pan > 0 {
			//			delete_lastpan := new(LastPan)
			//			del_result, _ := engine.Where("schedule_fid=? ", schedulefid).Delete(delete_lastpan)
			//			delete_schedule := new(Schedule)
			//			del_schedule_result, _ := engine.Where("schedule_fid=? ", schedulefid).Delete(delete_schedule)
			//			fmt.Println(del_result, del_schedule_result)
			return "开盘>0 或者即时盘 >0"
		}
		open_pan_desc := convToGB(tds_of_opentime_pan_table.Eq(1).Text())

		//		addLastPanLog()

		predict_result, predict_cmt := analysePanResult(open_pan, open_home_water, open_guest_water, real_pan, home_real_water, guest_real_water, home_pan_change_type, schedule_game_desc, schedulefid, cid)

		fmt.Println("float_open_pan")
		fmt.Println(open_home_water)
		fmt.Println("=====")

		fmt.Println("company:" + company)
		fmt.Println("home_pan_change_type:" + home_pan_change_type)
		fmt.Println("is big company:" + is_big_company)
		fmt.Println("change_time:" + change_time)
		fmt.Println("open_time:" + open_time)
		fmt.Println("flag:" + home_water_change_type + " " + home_water_up_down_flag)
		fmt.Println("home_real_water:", home_real_water)
		fmt.Println("home_real_water water sting:" + home_real_water_string)
		fmt.Println("guest_real_water:", guest_real_water)
		fmt.Println("guest_real_water water sting:" + guest_real_water_string)
		fmt.Println("pan:", real_pan, " ", real_pan_desc)

		fmt.Println("open_home_water water:", open_home_water)
		fmt.Println("open_guest_water water:", open_guest_water)
		fmt.Println("open pan:", open_pan, " ", open_pan_desc)

		exist_lastpan := new(LastPanEnd)
		has, _ := engine.Where("schedule_fid=? AND company_cid=? ", schedulefid, cid).Get(exist_lastpan)
		fmt.Println(has)
		if has {
			fmt.Println(company + "pan已存在!")
		} else {
			LastPan := new(LastPanEnd)
			LastPan.ScheduleNo = pname

			LastPan.ScheduleHome = schedule_home_name
			LastPan.ScheduleGuest = schedule_guest_name
			LastPan.ScheduleDate = schedule_date
			LastPan.ScheduleGameDesc = schedule_game_desc
			LastPan.CompanyCid = cid
			LastPan.CompanyName = company
			LastPan.ScheduleFid = schedulefid
			LastPan.OpenPan = open_pan
			LastPan.OpenPanDesc = open_pan_desc
			LastPan.OpenHomeWater = open_home_water
			LastPan.OpenGuestWater = open_guest_water
			LastPan.OpenPanTime = open_time
			LastPan.LastPan = real_pan
			LastPan.LastPanDesc = real_pan_desc
			LastPan.LastHomeWater = home_real_water
			LastPan.LastGuestWater = guest_real_water
			LastPan.LastChangeTime = change_time
			LastPan.LastHomePanChangeType = home_pan_change_type
			LastPan.IsBigCompany = is_big_company
			LastPan.LastHomeWaterChangeType = home_water_change_type
			LastPan.PredictResult = predict_result
			LastPan.PredictComment = predict_cmt

			ins_affected, ins_err := engine.Insert(LastPan)
			fmt.Println(ins_affected)
			fmt.Println(ins_err)

		}

		//		count_open_water := open_home_water +open_guest_water
		count_real_water := home_real_water + guest_real_water
		if count_real_water < 1.75 || count_real_water > 2 {
			delete_lastpan2 := new(LastPanEnd)
			delete2, _ := engine.Where("schedule_fid=? AND company_cid=? ", schedulefid, cid).Delete(delete_lastpan2)
			fmt.Println(delete2)
		}
	}
	return "成功"
}