func main() {
var url = "http://toutiao.io"
p, err := goquery.ParseUrl(url)
if err != nil {
panic(err)
}
// toutiao.io
title := p.Find("title").Text()
fmt.Println(title)
t := p.Find(".title a")
for i := 0; i < t.Length(); i++ {
d := t.Eq(i).Text()
l := t.Eq(i).Attr("href")
c := p.Find(".summary a").Eq(i).Text()
fmt.Println(l, d, "||", c)
}
// geek.csdn.net
p, err = goquery.ParseUrl("http://geek.csdn.net/hotest")
if err != nil {
panic(err)
}
title = p.Find("title").Text()
t = p.Find("a.title")
fmt.Println(title)
for i := 0; i < t.Length(); i++ {
l := t.Eq(i).Attr("href")
c := t.Eq(i).Text()
fmt.Println(l, c)
}
// ituring.com
p, err = goquery.ParseUrl("http://www.ituring.com.cn/")
if err != nil {
panic(err)
}
title = p.Find("title").Text()
t = p.Find(".arc-list").Eq(0)
x, _ := goquery.ParseString(t.Html())
t = x.Find("dt a")
fmt.Println(title)
for i := 0; i < t.Length(); i++ {
c := t.Eq(i).Text()
l := t.Eq(i).Attr("href")
fmt.Printf("http://www.ituring.com.cn/%s %s\n", l, c)
}
}
func GetDownloadLinks(url string) {
/*
*通过传入的书籍URL地址；提取到下载地址
*提取书籍的名称和url地址
*
*/
fmt.Println("\n")
fmt.Println("书籍地址", url)
r, err := goquery.ParseUrl(url)
if err != nil {
panic(err)
} else {
text := r.Find(".download-link")
bookName := r.Find("h1").Text()
fmt.Println("书籍名称:", bookName)
//fmt.Println(text)
for i := 0; i < text.Length(); i++ {
downloadlink := text.Eq(i).Attr("href")
fmt.Println("下载地址", downloadlink)
DownloadBook(downloadlink, bookName)
}
}
}
func ParsePanMap(date string, history bool) {
bet_url := strings.Replace(myinit.DateUrl, "TTT", date, -1)
fmt.Println("bet_url:", bet_url)
pan_url := myinit.PanUrl
html_obj, _ := goquery.ParseUrl(bet_url)
schedule_trs := html_obj.Find(".bet_table tbody tr")
for i, _ := range schedule_trs {
schedule_int_info := make(map[string]int)
schedule_string_info := make(map[string]string)
// insert schedule
fid, _ := strconv.Atoi(schedule_trs.Eq(i).Attr("fid"))
schedule_int_info["schedule_fenxi_id"] = int(fid)
schedule_string_info["schedule_home"] = common.ConvToGB(schedule_trs.Eq(i).Attr("homesxname"))
schedule_string_info["schedule_guest"] = common.ConvToGB(schedule_trs.Eq(i).Attr("awaysxname"))
schedule_string_info["schedule_date"] = schedule_trs.Eq(i).Attr("pdate")
schedule_pan_url := strings.Replace(pan_url, "TTT", strconv.Itoa(fid), -1)
GetPanValue(schedule_pan_url, fid, schedule_string_info, date)
}
}
func checkPanUseable(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)
odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
if odds_tr.Length() == 0 {
return false
}
for i := 0; i < odds_tr.Length(); i++ {
tr_item := odds_tr.Eq(i)
td_of_company := tr_item.Find("td").Eq(1)
if td_of_company.Find("p a").Attr("title") == "" {
continue
}
is_useable := checkPanTr(tr_item, schedule_fenxi_id)
if is_useable == false {
return false
}
}
if odds_tr.Length() >= 30 {
ajax_res := checkPanUseableFromAjax(30, schedule_fenxi_id)
if ajax_res == false {
return false
}
}
return true
}
func ParseBetUrl(date string, history bool) {
bet_url := getBetUrl(date)
html_obj, _ := goquery.ParseUrl(bet_url)
schedule_trs := html_obj.Find(".bet_table tbody tr")
for i, _ := range schedule_trs {
is_end := schedule_trs.Eq(i).Attr("isend")
if is_end == "1" && history == false {
continue
}
today := common.GetToday()
schedule_is_today := today == schedule_trs.Eq(i).Attr("pdate")
// fmt.Println("schedule_is_today:===",schedule_is_today)
if schedule_is_today == false && history == false {
// continue
}
schedule_int_info, schedule_string_info := pareseScheduleTR(schedule_trs.Eq(i))
schedule.Add(schedule_int_info, schedule_string_info)
// end insert schedule
//parse pan data
res := ParsePanByScheduleFenxiId(schedule_int_info["schedule_fenxi_id"], date, schedule_string_info)
if res == false {
continue
}
//计算预测比率
calcScheduleResult(schedule_int_info, schedule_string_info)
// return
}
}
func GetZhihuQuestionList(url string) {
/*
* 通过传入的url、分析提取url里面的问题列表
* 用for循环提交分页URL地址
* 并用goquery查找页面内容是否存在、用以判断是否还有分页
* 用提取到的短URL加上统一URL地址头得到某一个完整URL地址
*/
urlHeader := "http://www.zhihu.com"
for i := 1; i < 100; i++ {
url := url + strconv.Itoa(i)
fmt.Println(url)
r, err := goquery.ParseUrl(url)
if err != nil {
panic(err)
} else {
text := r.Find(".zm-item-title a") //查找所有问题列表
if text.Length() > 0 {
//取到text里面的所有"href"属性的数据
for i := 0; i < text.Length(); i++ {
singleUrl := urlHeader + text.Eq(i).Attr("href")
urlList = append(urlList, singleUrl)
GetSubjectBody(singleUrl)
}
} else { //如果text的长度小于0表示没有找到
fmt.Print(".........没有文章了.......\n")
break
}
}
}
}
func ParseBetUrl(date string, history bool) {
bet_url := getBetUrl(date)
html_obj, _ := goquery.ParseUrl(bet_url)
schedule_trs := html_obj.Find(".bet_table tbody tr")
for i, _ := range schedule_trs {
is_end := schedule_trs.Eq(i).Attr("isend")
if is_end == "1" && history == false {
continue
}
schedule_int_info, schedule_string_info := pareseScheduleTR(schedule_trs.Eq(i), date)
//if(schedule_int_info["schedule_fenxi_id"]!=556793){
// continue
//}
//parse pan data
res := ParsePanByScheduleFenxiId(schedule_int_info["schedule_fenxi_id"], date, schedule_string_info)
if res == false {
continue
}
schedule.Add(schedule_int_info, schedule_string_info)
//计算预测比率
calcScheduleResult(schedule_int_info, schedule_string_info)
// return
}
}
func GetZi5PageUrl(category string) {
/*
*通过传入的分类标签；得到分类地址
*用For循环传递分页页码；然后用goquery得到单个书籍的URL地址；
*如果分页里面得不到值就认为分页结束
*
*
*/
url := "http://book.zi5.me/archives/book-gentre/"
//var category string
for i := 1; i < 100; i++ {
url := url + category + "/page/" + strconv.Itoa(i) //分页地址
fmt.Println("\n")
fmt.Println("分页地址", i, url)
r, err := goquery.ParseUrl(url)
if err != nil {
panic(err)
} else {
//查找<class="thumb-holder" > 和<a
text := r.Find(".thumb-holder a")
//fmt.Println(text)
if text.Length() > 0 {
//取到text里面的所有"href"属性的数据
for x := 0; x < text.Length(); x++ {
bookUrl := text.Eq(x).Attr("href")
GetDownloadLinks(bookUrl)
}
} else { //如果text的长度小于0表示没有找到
fmt.Print("本分类下已经没有书籍\n")
break
}
}
}
}
func main() {
x, err := goquery.ParseUrl("http://www.youtube.com/watch?v=ob_nh1WMMzU")
if err != nil {
panic(err)
}
x.Find("#eow-title").Print()
fmt.Println("---")
x, err = goquery.ParseUrl("http://thepiratebay.se/search/one%20day%202011/0/99/0")
if err != nil {
panic(err)
}
x.Find("a.detLink").Print()
fmt.Println("---")
for _, v := range x.Find("a.detLink").HtmlAll() {
fmt.Println(v)
}
}
func main() {
pool := mongo.NewDialPool("localhost:27018", 1000)
var i int = 1
chs := make([]chan bool, 50)
hasMore := true
for ; i <= 50; i++ {
ch := make(chan bool)
chs[i-1] = ch
go func(i int, ch chan bool) {
conn, _ := pool.Get()
db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
coll := db.C("foods")
data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
if nil != err {
log.Fatalln("response fail ,", err)
ch <- false
return
}
nodes := data.Find("div.lp_result_list")
size := nodes.Find("li").Length()
if size <= 0 {
hasMore = false
}
for idx := 0; idx < size; idx++ {
item := nodes.Find("li").Eq(idx)
link := item.Find("a")
href := link.Attr("href")
name := link.Attr("title")
img := item.Find("img").Attr("src")
// log.Println(name, "|", href, "|", img)
if len(name) > 0 {
err := coll.Upsert(mongo.M{"name": name}, mongo.M{"name": name, "img_url": img, "link": href})
// log.Println("insert mongo|", err, "|", href)
log.Println("err", err, "name:", name, "link:", link, "href:", href, "img", img)
}
}
ch <- true
log.Println(i)
}(i, ch)
}
for i, val := range chs {
<-val
log.Println("end:", i)
}
}
func ParseResultUrl(date string, history bool) (res bool) {
if date == "" {
return false
}
result_url := strings.Replace(myinit.ResultUrl, "DDD", date, -1)
fmt.Println(result_url)
schedule_string_info := make(map[string]string)
schedule_float_info := make(map[string]float32)
pan_string_info := make(map[string]string)
pan_float_info := make(map[string]float32)
html_obj, _ := goquery.ParseUrl(result_url)
schedule_trs := html_obj.Find(".ld_table tbody tr")
for i, _ := range schedule_trs {
if i == 0 {
continue
}
tr := schedule_trs.Eq(i)
tds := tr.Find("td")
schedule_string_info["schedule_result_no"] = common.ConvToGB(tds.Eq(0).Html())
schedule_string_info["schedule_score"] = common.ConvToGB(tds.Eq(6).Html())
schedule_string_info["schedule_spf_result"] = common.ConvToGB(tds.Eq(11).Html())
schedule_float_info["schedule_spf_odd"] = common.ConvToFloat32(tds.Eq(12).Text())
schedule_string_info["schedule_rqspf_result"] = common.ConvToGB(tds.Eq(8).Html())
schedule_float_info["schedule_rqspf_odd"] = common.ConvToFloat32(tds.Eq(9).Text())
schedule_string_info["schedule_zjq_result"] = common.ConvToGB(tds.Eq(14).Html())
schedule_float_info["schedule_zjq_odd"] = common.ConvToFloat32(tds.Eq(15).Text())
schedule_string_info["schedule_bqc_result"] = common.ConvToGB(tds.Eq(17).Html())
schedule_float_info["schedule_bqc_odd"] = common.ConvToFloat32(tds.Eq(18).Text())
pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
pan_string_info["schedule_score"] = schedule_string_info["schedule_score"]
pan_string_info["schedule_spf_result"] = schedule_string_info["schedule_spf_result"]
pan_string_info["schedule_rqspf_result"] = schedule_string_info["schedule_rqspf_result"]
pan_string_info["schedule_zjq_result"] = schedule_string_info["schedule_zjq_result"]
pan_string_info["schedule_bqc_result"] = schedule_string_info["schedule_bqc_result"]
has := schedule.CheckExistsByResultNoAndDate(schedule_string_info["schedule_result_no"], date)
fmt.Println("has:", has)
if has == false {
continue
}
schedule.UpdateScheduleResult(date, schedule_float_info, schedule_string_info)
asiapan.UpdateAsiaPanResult(date, schedule_float_info, pan_string_info)
asiapanlog.UpdateAsiaPanResult(date, pan_float_info, pan_string_info)
}
return true
}
func GetSubjectBody(url string) {
/*
* 通过传入的单个问题url、分析提取url里面的问题标题和内容
* 用goquery查找页面提取里面title作为邮件的subject、用html()作为body
*
*/
//var url = "http://www.zhihu.com/question/24859069"
p, error := goquery.ParseUrl(url)
if error != nil {
panic(error)
}
subject := p.Find("title").Text()
body := p.Html()
fmt.Print(subject)
GetConf(subject, body)
}
func main() {
pool := mongo.NewDialPool("localhost:27018", 1000)
var i int = 1
ch := make([]chan bool, 20)
for ; i <= 20; i++ {
ch[i-1] = make(chan bool)
go func(i int, ch chan bool) {
conn, _ := pool.Get()
db := &mongo.Database{conn, "meishi", mongo.DefaultLastErrorCmd}
coll := db.C("foods")
data, err := goquery.ParseUrl(targetUrl + strconv.Itoa(i))
if nil != err {
log.Fatalln("response fail ,", err)
ch <- false
return
}
nodes := data.Find("#page-" + strconv.Itoa(i))
size := nodes.Find("div.media").Length()
for idx := 0; idx < size; idx++ {
item := nodes.Find("div.media").Eq(idx)
h2 := item.Find("h2")
link := h2.Find("a")
href := link.Attr("href")
name := link.Attr("title")
img := item.Find("img").Attr("src")
// log.Println(name, "|", href, "|", img)
if len(name) > 0 {
err := coll.Insert(mongo.M{"name": name, "img_url": img, "link": href})
log.Println("insert mongo|", err, "|", href)
}
}
ch <- true
log.Println(i)
}(i, ch[i-1])
}
for i, val := range ch {
<-val
log.Println("end:", i)
}
}
func doParsePanUrl(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
pan_int_info := make(map[string]int)
pan_float_info := make(map[string]float32)
pan_string_info := make(map[string]string)
pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)
schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
home_td := schedule_item.Eq(0)
guest_td := schedule_item.Eq(4)
center_td := schedule_item.Eq(2)
pan_int_info["schedule_fenxi_id"] = schedule_fenxi_id
pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
pan_string_info["schedule_no"] = schedule_string_info["schedule_no"]
pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
pan_string_info["schedule_league"] = schedule_string_info["schedule_league"]
pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())
pan_string_info["schedule_guest"] = common.ConvToGB(guest_td.Find("ul li a").Text())
pan_string_info["schedule_game_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
pan_string_info["schedule_date_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .game_time ").Text())
odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
for i := 0; i < odds_tr.Length(); i++ {
tr_item := odds_tr.Eq(i)
td_of_company := tr_item.Find("td").Eq(1)
if td_of_company.Find("p a").Attr("title") == "" {
continue
}
parse_res := doParsePanTr(tr_item, schedule_fenxi_id, pan_int_info, pan_float_info, pan_string_info, false)
if parse_res == false {
return false
}
}
if odds_tr.Length() >= 30 {
ajax_res := ParsePanUrlFromAjax(30, schedule_fenxi_id, pan_string_info)
if ajax_res == false {
return false
}
}
return true
}
// use Goquery to fetch product-name from html
func fetchProductName(url string) {
p, err := goquery.ParseUrl(url)
if err != nil {
panic(err)
} else {
pTitle := p.Find("title").Text() // fetch the content of title
fmt.Println(pTitle)
productList := p.Find(".product-name")
priceList := p.Find(".product_price")
for i := 0; i < productList.Length(); i++ {
product := productList.Eq(i).Text()
price := priceList.Eq(i).Text()
fmt.Println(product + ": " + price)
}
}
}
func GetQuery(getUrl string) string {
x, err := goquery.ParseUrl(getUrl)
if err != nil {
panic(err)
return "ERROR! IN Get Query."
}
//x.Find(".info").Print()
detail := x.Find(".content").Html()
//格式化处理
//移除HTML
r := regexp.MustCompile("<.*?>")
detail = r.ReplaceAllString(detail, " ")
//移除换行
r = regexp.MustCompile("[ \f\n\r\t\v]")
detail = r.ReplaceAllString(detail, "")
return detail
}
func getHmtl(url string) (string, error) {
p, err := goquery.ParseUrl(url)
if err != nil {
return "", err
} else {
t := p.Find(".lyric-action")
if t.Length() > 0 {
for i := 0; i < 1; i++ { //只返回一条记录
if strings.Contains(t.Html(), "down-lrc-btn") {
lrc := strings.Split(strings.SplitN(t.Html(), "down-lrc-btn", 2)[1], "'")
return lrc[3], nil
}
}
}
}
return "", errors.New("没能找到你需要的歌词")
}
func GetMusicID(name string) []interface{} {
p, err := goquery.ParseUrl(url_music + name + url_music_break)
if err != nil {
fmt.Println(err)
} else {
var items []interface{}
t := p.Find(".song-item-hook")
if t.Length() > 0 {
for i := 0; i < t.Length(); i++ {
if strings.Contains(t.Eq(i).Html(), "data-songdata") {
str := strings.SplitN(t.Eq(i).Html(), "data-songdata", 2)[0]
str = strings.SplitN(str, "href=", 2)[1]
if strings.Contains(str, " ") {
str = strings.Split(str, " ")[0]
} else {
continue
}
if strings.Contains(str, "/") {
item_ht := strings.Split(str, "/")
str = item_ht[len(item_ht)-1]
} else {
continue
}
if strings.Contains(str, "\"") {
str = strings.Split(str, "\"")[0]
} else {
continue
}
items = append(items, str)
}
}
}
return items
}
return nil
}
func ParsePanUrl(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
pan_int_info := make(map[string]int)
pan_float_info := make(map[string]float32)
pan_string_info := make(map[string]string)
pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)
schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
home_td := schedule_item.Eq(0)
guest_td := schedule_item.Eq(4)
center_td := schedule_item.Eq(2)
pan_int_info["schedule_fenxi_id"] = schedule_fenxi_id
pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
pan_string_info["schedule_no"] = schedule_string_info["schedule_no"]
pan_string_info["schedule_result_no"] = schedule_string_info["schedule_result_no"]
pan_string_info["schedule_league"] = schedule_string_info["schedule_league"]
pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())
pan_string_info["schedule_guest"] = common.ConvToGB(guest_td.Find("ul li a").Text())
pan_string_info["schedule_game_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
pan_string_info["schedule_date_desc"] = common.ConvToGB(center_td.Find(".odds_hd_center .game_time ").Text())
odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
for i := 0; i < odds_tr.Length(); i++ {
tr_item := odds_tr.Eq(i)
td_of_company := tr_item.Find("td").Eq(1)
if td_of_company.Find("p a").Attr("title") == "" {
continue
}
company_id := tr_item.Attr("id")
pan_string_info["company_id"] = company_id
pan_string_info["company_name"] = common.ConvToGB(td_of_company.Find("p a").Attr("title"))
var is_big_company = 0
if td_of_company.Find("p img").Attr("src") == "" {
is_big_company = 0
} else {
is_big_company = 1
fmt.Println("src:" + td_of_company.Find("p img").Attr("src"))
}
pan_int_info["is_big_company"] = is_big_company
table_of_pan_detail := tr_item.Find("td .pl_table_data")
table_of_opentime_pan := table_of_pan_detail.Eq(1)
tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")
open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
pan_float_info["open_pan"] = float32(open_pan_32)
pan_string_info["open_pan_desc"] = common.ConvToGB(tds_of_opentime_pan_table.Eq(1).Text())
open_home_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(0).Text(), 32)
open_guest_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(2).Text(), 32)
pan_float_info["open_home_water"] = float32(open_home_water_32)
pan_float_info["open_guest_water"] = float32(open_guest_water_32)
td_of_pan_time := tr_item.Find("td time")
pan_string_info["open_pan_time"] = td_of_pan_time.Eq(1).Text()
table_of_realtime_pan := table_of_pan_detail.Eq(0)
tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")
real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
pan_float_info["real_pan"] = float32(real_pan_32)
pan_string_info["real_pan_desc"] = common.ConvToGB(tds_of_realtime_pan_table.Eq(1).Text())
home_real_water_string := common.ConvToGB(tds_of_realtime_pan_table.Eq(0).Text())
home_real_water_str := strings.Replace(home_real_water_string, "↑", "", -1)
home_real_water_str = strings.Replace(home_real_water_str, "↓", "", -1)
guest_real_water_string := common.ConvToGB(tds_of_realtime_pan_table.Eq(2).Text())
guest_real_water_str := strings.Replace(guest_real_water_string, "↑", "", -1)
guest_real_water_str = strings.Replace(guest_real_water_str, "↓", "", -1)
home_real_water_32, _ := strconv.ParseFloat(home_real_water_str, 32)
guest_real_water_32, _ := strconv.ParseFloat(guest_real_water_str, 32)
pan_float_info["real_home_water"] = float32(home_real_water_32)
pan_float_info["real_guest_water"] = float32(guest_real_water_32)
fmt.Println("before home_real_water:", pan_float_info["real_home_water"])
pan_string_info["pan_change_time"] = td_of_pan_time.Eq(0).Text()
td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
home_pan_change_type := common.ConvToGB(td_item_of_real_pan.Find("font").Text())
home_pan_change_type = strings.TrimSpace(home_pan_change_type)
pan_int_info["home_pan_change_type"] = 0
if home_pan_change_type == "升" {
pan_int_info["home_pan_change_type"] = 1
pan_string_info["home_pan_change_type_desc"] = home_pan_change_type
}
if home_pan_change_type == "降" {
pan_int_info["home_pan_change_type"] = -1
pan_string_info["home_pan_change_type_desc"] = home_pan_change_type
}
home_water_up_down_flag := tds_of_realtime_pan_table.Eq(0).Attr("class")
pan_int_info["home_water_change_type"] = 0
if home_water_up_down_flag == "ping" {
pan_int_info["home_water_change_type"] = -1 // down
pan_string_info["home_water_change_type_desc"] = "水位降" // down
}
if home_water_up_down_flag == "ying" {
pan_int_info["home_water_change_type"] = 1 // up
pan_string_info["home_water_change_type_desc"] = "水位升" // up
}
fmt.Println("before2 home_real_water:", pan_float_info["real_home_water"])
real_pan_string := strings.Replace(pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"], "", -1)
real_pan_desc := strings.TrimSpace(real_pan_string)
// panmap.Add(pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
// panmap.Add(real_pan_desc, pan_float_info["real_pan"])
fmt.Println("date:", pan_string_info["schedule_date"], pan_string_info["schedule_home"])
fmt.Println("open:", pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
fmt.Println("real:", real_pan_desc, pan_float_info["real_pan"])
parse_change_data := ParsePanChangeUrl(schedule_fenxi_id, company_id, pan_int_info, pan_float_info, pan_string_info)
if parse_change_data == false {
continue
}
fmt.Println("before3 home_real_water:", pan_float_info["real_home_water"])
if pan_float_info["open_pan"] > 0 || pan_float_info["real_pan"] > 0 {
delete_asiapan := new(myinit.AsiaPan)
del_result, _ := myinit.Engine.Where("schedule_fenxi_id=? ", schedule_fenxi_id).Delete(delete_asiapan)
delete_schedule := new(myinit.Schedule)
del_schedule_result, _ := myinit.Engine.Where("schedule_fenxi_id=? ", schedule_fenxi_id).Delete(delete_schedule)
fmt.Println(del_result, del_schedule_result)
fmt.Println("开盘>0 或者即时盘 >0")
return false
}
predict1_result, predict1_cmt := analyse.AnalysePanResult1(pan_int_info, pan_float_info, pan_string_info)
pan_string_info["predict1_result"] = predict1_result
pan_string_info["predict1_cmt"] = predict1_cmt
predict2_result, predict2_cmt := analyse.AnalysePanResult2(pan_int_info, pan_float_info, pan_string_info)
pan_string_info["predict2_result"] = predict2_result
pan_string_info["predict2_cmt"] = predict2_cmt
// fmt.Println("company:" + company)
// fmt.Println("home_pan_change_type:" + home_pan_change_type)
// fmt.Println("is big company:" + is_big_company)
// fmt.Println("change_time:" + change_time)
// fmt.Println("open_time:" + open_time)
// fmt.Println("flag:" + home_water_change_type + " " + home_water_up_down_flag)
fmt.Println("home_real_water:", pan_float_info["real_home_water"])
fmt.Println(home_real_water_32)
fmt.Println(float32(home_real_water_32))
fmt.Println("home_real_water water sting:" + home_real_water_str + home_real_water_string)
fmt.Println("guest_real_water:", pan_float_info["real_guest_water"])
fmt.Println("guest_real_water water sting:" + guest_real_water_str + guest_real_water_string)
fmt.Println("pan:", pan_float_info["real_pan"], " ", real_pan_desc)
// fmt.Println("open_home_water water:", open_home_water)
// fmt.Println("open_guest_water water:", open_guest_water)
// fmt.Println("open pan:", open_pan, " ", open_pan_desc)
exist_asiapan := new(myinit.AsiaPan)
has, _ := myinit.Engine.Where("schedule_fenxi_id=? AND company_id=? ", schedule_fenxi_id, company_id).Get(exist_asiapan)
if has {
fmt.Println(pan_string_info["company_name"] + "pan已存在！")
if exist_asiapan.PanChangeTime != pan_string_info["pan_change_time"] {
fmt.Println(pan_string_info["company_name"] + "pan有变化！")
asiapan.UpdateAsiaPanInfo(pan_int_info, pan_float_info, pan_string_info)
}
} else {
asiapan.Add(pan_int_info, pan_float_info, pan_string_info)
}
// count_open_water := open_home_water +open_guest_water
count_real_water := pan_float_info["real_home_water"] + pan_float_info["real_guest_water"]
if count_real_water < 1.75 || count_real_water > 2 {
// fmt.Println("+++++++")
// fmt.Println(pan_float_info["real_home_water"])
// fmt.Println(pan_float_info["real_guest_water"])
// delete_asiapan2 := new(myinit.AsiaPan)
// delete2, _ := myinit.Engine.Where("schedule_fenxi_id=? AND company_id=? ", schedule_fenxi_id, company_id).Delete(delete_asiapan2)
// fmt.Println(delete2)
}
}
if odds_tr.Length() >= 30 {
ParsePanUrlFromAjax(30, schedule_fenxi_id, pan_string_info)
}
return true
}
func ParseBetUrl(date string, history bool) {
bet_url := strings.Replace(myinit.DateUrl, "TTT", date, -1)
if date == "" {
bet_url = myinit.IndexUrl
}
fmt.Println("bet_url:", bet_url)
pan_url := myinit.PanUrl
html_obj, _ := goquery.ParseUrl(bet_url)
// fmt.Println(html_obj.HtmlAll())
schedule_trs := html_obj.Find(".bet_table tbody tr")
for i, _ := range schedule_trs {
is_end := schedule_trs.Eq(i).Attr("isend")
if is_end == "1" && history == false {
// fmt.Println("is_end")
continue
}
schedule_int_info := make(map[string]int)
schedule_string_info := make(map[string]string)
// insert schedule
fid, _ := strconv.Atoi(schedule_trs.Eq(i).Attr("fid"))
schedule_int_info["schedule_fenxi_id"] = int(fid)
schedule_string_info["schedule_home"] = common.ConvToGB(schedule_trs.Eq(i).Attr("homesxname"))
schedule_string_info["schedule_guest"] = common.ConvToGB(schedule_trs.Eq(i).Attr("awaysxname"))
schedule_string_info["schedule_date"] = schedule_trs.Eq(i).Attr("pdate")
schedule_string_info["schedule_league"] = common.ConvToGB(schedule_trs.Eq(i).Attr("lg"))
schedule_string_info["schedule_week_day"] = common.ConvToGB(schedule_trs.Eq(i).Attr("gdate"))
schedule_string_info["schedule_no"] = schedule_trs.Eq(i).Attr("pname")
schedule_string_info["schedule_rq_num"] = schedule_trs.Eq(i).Attr("rq")
week_date := schedule_string_info["schedule_no"][0:1]
schedule_string_info["schedule_result_no"] = myinit.WeekDesc[week_date] + schedule_string_info["schedule_no"][1:]
schedule_string_info["schedule_bet_end_time"] = schedule_trs.Eq(i).Attr("pendtime")
today := time.Now().Format("2006-01-02")
schedule_is_today := today == schedule_string_info["schedule_date"]
// fmt.Println("schedule_is_today:===",schedule_is_today)
if schedule_is_today == false && history == false {
continue
}
schedule.Add(schedule_int_info, schedule_string_info)
// end insert schedule
schedule_pan_url := strings.Replace(pan_url, "TTT", strconv.Itoa(fid), -1)
// fmt.Println("schedule_pan_url:",schedule_pan_url)
res := ParsePanUrl(schedule_pan_url, fid, schedule_string_info, date)
fmt.Println(res)
fmt.Println("---------")
fmt.Println(schedule_string_info["schedule_date"])
fmt.Println(schedule_string_info["schedule_no"])
fmt.Println("---------")
if res == false {
continue
}
calcScheduleResult(schedule_int_info, schedule_string_info)
return
}
}
func GetPanValue(schedule_pan_url string, schedule_fenxi_id int, schedule_string_info map[string]string, date string) (res bool) {
pan_int_info := make(map[string]int)
pan_float_info := make(map[string]float32)
pan_string_info := make(map[string]string)
pan_html_obj, _ := goquery.ParseUrl(schedule_pan_url)
schedule_item := pan_html_obj.Find(".odds_hd_cont table tbody tr td")
home_td := schedule_item.Eq(0)
pan_string_info["schedule_date"] = schedule_string_info["schedule_date"]
pan_string_info["schedule_home"] = common.ConvToGB(home_td.Find("ul li a").Text())
odds_tr := pan_html_obj.Find(".table_cont table tbody tr")
for i := 0; i < odds_tr.Length(); i++ {
tr_item := odds_tr.Eq(i)
td_of_company := tr_item.Find("td").Eq(1)
if td_of_company.Find("p a").Attr("title") == "" {
continue
}
pan_string_info["company_name"] = common.ConvToGB(td_of_company.Find("p a").Attr("title"))
table_of_pan_detail := tr_item.Find("td .pl_table_data")
table_of_opentime_pan := table_of_pan_detail.Eq(1)
tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")
open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
pan_float_info["open_pan"] = float32(open_pan_32)
pan_string_info["open_pan_desc"] = common.ConvToGB(tds_of_opentime_pan_table.Eq(1).Text())
table_of_realtime_pan := table_of_pan_detail.Eq(0)
tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")
real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
pan_float_info["real_pan"] = float32(real_pan_32)
pan_string_info["real_pan_desc"] = common.ConvToGB(tds_of_realtime_pan_table.Eq(1).Text())
td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
home_pan_change_type := common.ConvToGB(td_item_of_real_pan.Find("font").Text())
home_pan_change_type = strings.TrimSpace(home_pan_change_type)
pan_int_info["home_pan_change_type"] = 0
if home_pan_change_type == "升" {
pan_int_info["home_pan_change_type"] = 1
pan_string_info["home_pan_change_type_desc"] = home_pan_change_type
}
if home_pan_change_type == "降" {
pan_int_info["home_pan_change_type"] = -1
pan_string_info["home_pan_change_type_desc"] = home_pan_change_type
}
real_pan_string := strings.Replace(pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"], "", -1)
real_pan_desc := strings.TrimSpace(real_pan_string)
fmt.Println("====here====")
fmt.Println(i)
fmt.Println("date:", pan_string_info["schedule_date"], pan_string_info["schedule_home"], pan_string_info["company_name"])
// fmt.Println("open:", pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
// fmt.Println("real:", real_pan_desc, pan_float_info["real_pan"])
// fmt.Println("real desc:", pan_string_info["real_pan_desc"], pan_string_info["home_pan_change_type_desc"])
fmt.Println("++end here+++")
panmap.Add(pan_string_info["open_pan_desc"], pan_float_info["open_pan"])
panmap.Add(real_pan_desc, pan_float_info["real_pan"])
}
if odds_tr.Length() >= 30 {
FetchTrByAjax(30, schedule_fenxi_id, pan_string_info["schedule_date"], pan_string_info["schedule_home"])
}
return true
}
func ParseOddUrl(schedule_odds_url string, schedulefid string, pname string, schedule_date_orig string) (a string) {
odds_html, _ := goquery.ParseUrl(schedule_odds_url)
schedule_item := odds_html.Find(".odds_hd_cont table tbody tr td")
home_td := schedule_item.Eq(0)
guest_td := schedule_item.Eq(4)
center_td := schedule_item.Eq(2)
schedule_home_name := convToGB(home_td.Find("ul li a").Text())
schedule_guest_name := convToGB(guest_td.Find("ul li a").Text())
schedule_game_desc := convToGB(center_td.Find(".odds_hd_center .odds_hd_ls a").Text())
// schedule_date := convToGB(center_td.Find(".odds_hd_center .game_time ").Text())
schedule_date := schedule_date_orig
odds_tr := odds_html.Find(".table_cont table tbody tr")
for i := 0; i < odds_tr.Length(); i++ {
tr_item := odds_tr.Eq(i)
td_of_company := tr_item.Find("td").Eq(1)
company := convToGB(td_of_company.Find("p a").Attr("title"))
if td_of_company.Find("p a").Attr("title") == "" {
continue
}
var is_big_company = "0"
if td_of_company.Find("p img").Attr("src") == "" {
is_big_company = "0"
} else {
is_big_company = "1"
fmt.Println("src:" + td_of_company.Find("p img").Attr("src"))
}
cid := tr_item.Attr("id")
td_of_pan_time := tr_item.Find("td time")
change_time := td_of_pan_time.Eq(0).Text()
open_time := td_of_pan_time.Eq(1).Text()
table_of_pan_detail := tr_item.Find("td .pl_table_data")
table_of_realtime_pan := table_of_pan_detail.Eq(0)
tds_of_realtime_pan_table := table_of_realtime_pan.Find("tbody tr td")
home_water_up_down_flag := tds_of_realtime_pan_table.Eq(0).Attr("class")
var home_water_change_type = "water_unknown"
if home_water_up_down_flag == "ping" {
home_water_change_type = "water_down" // down
}
if home_water_up_down_flag == "ying" {
home_water_change_type = "water_up" // up
}
home_real_water_string := convToGB(tds_of_realtime_pan_table.Eq(0).Text())
guest_real_water_string := convToGB(tds_of_realtime_pan_table.Eq(2).Text())
real_pan_32, _ := strconv.ParseFloat(tds_of_realtime_pan_table.Eq(1).Attr("ref"), 32)
real_pan := float32(real_pan_32)
real_pan_desc := convToGB(tds_of_realtime_pan_table.Eq(1).Text())
td_item_of_real_pan := tds_of_realtime_pan_table.Eq(1)
home_pan_change_type := convToGB(td_item_of_real_pan.Find("font").Text())
home_pan_change_type = strings.TrimSpace(home_pan_change_type)
home_real_water_str := strings.Replace(home_real_water_string, "↑", "", -1)
home_real_water_str = strings.Replace(home_real_water_str, "↓", "", -1)
guest_real_water_str := strings.Replace(guest_real_water_string, "↑", "", -1)
guest_real_water_str = strings.Replace(guest_real_water_str, "↓", "", -1)
table_of_opentime_pan := table_of_pan_detail.Eq(1)
tds_of_opentime_pan_table := table_of_opentime_pan.Find("tbody tr td")
open_home_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(0).Text(), 32)
open_guest_water_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(2).Text(), 32)
home_real_water_32, _ := strconv.ParseFloat(home_real_water_str, 32)
guest_real_water_32, _ := strconv.ParseFloat(guest_real_water_str, 32)
open_home_water := float32(open_home_water_32)
open_guest_water := float32(open_guest_water_32)
home_real_water := float32(home_real_water_32)
guest_real_water := float32(guest_real_water_32)
open_pan_32, _ := strconv.ParseFloat(tds_of_opentime_pan_table.Eq(1).Attr("ref"), 32)
open_pan := float32(open_pan_32)
if open_pan > 0 || real_pan > 0 {
// delete_lastpan := new(LastPan)
// del_result, _ := engine.Where("schedule_fid=? ", schedulefid).Delete(delete_lastpan)
// delete_schedule := new(Schedule)
// del_schedule_result, _ := engine.Where("schedule_fid=? ", schedulefid).Delete(delete_schedule)
// fmt.Println(del_result, del_schedule_result)
return "开盘>0 或者即时盘 >0"
}
open_pan_desc := convToGB(tds_of_opentime_pan_table.Eq(1).Text())
// addLastPanLog()
predict_result, predict_cmt := analysePanResult(open_pan, open_home_water, open_guest_water, real_pan, home_real_water, guest_real_water, home_pan_change_type, schedule_game_desc, schedulefid, cid)
fmt.Println("float_open_pan")
fmt.Println(open_home_water)
fmt.Println("=====")
fmt.Println("company:" + company)
fmt.Println("home_pan_change_type:" + home_pan_change_type)
fmt.Println("is big company:" + is_big_company)
fmt.Println("change_time:" + change_time)
fmt.Println("open_time:" + open_time)
fmt.Println("flag:" + home_water_change_type + " " + home_water_up_down_flag)
fmt.Println("home_real_water:", home_real_water)
fmt.Println("home_real_water water sting:" + home_real_water_string)
fmt.Println("guest_real_water:", guest_real_water)
fmt.Println("guest_real_water water sting:" + guest_real_water_string)
fmt.Println("pan:", real_pan, " ", real_pan_desc)
fmt.Println("open_home_water water:", open_home_water)
fmt.Println("open_guest_water water:", open_guest_water)
fmt.Println("open pan:", open_pan, " ", open_pan_desc)
exist_lastpan := new(LastPanEnd)
has, _ := engine.Where("schedule_fid=? AND company_cid=? ", schedulefid, cid).Get(exist_lastpan)
fmt.Println(has)
if has {
fmt.Println(company + "pan已存在！")
} else {
LastPan := new(LastPanEnd)
LastPan.ScheduleNo = pname
LastPan.ScheduleHome = schedule_home_name
LastPan.ScheduleGuest = schedule_guest_name
LastPan.ScheduleDate = schedule_date
LastPan.ScheduleGameDesc = schedule_game_desc
LastPan.CompanyCid = cid
LastPan.CompanyName = company
LastPan.ScheduleFid = schedulefid
LastPan.OpenPan = open_pan
LastPan.OpenPanDesc = open_pan_desc
LastPan.OpenHomeWater = open_home_water
LastPan.OpenGuestWater = open_guest_water
LastPan.OpenPanTime = open_time
LastPan.LastPan = real_pan
LastPan.LastPanDesc = real_pan_desc
LastPan.LastHomeWater = home_real_water
LastPan.LastGuestWater = guest_real_water
LastPan.LastChangeTime = change_time
LastPan.LastHomePanChangeType = home_pan_change_type
LastPan.IsBigCompany = is_big_company
LastPan.LastHomeWaterChangeType = home_water_change_type
LastPan.PredictResult = predict_result
LastPan.PredictComment = predict_cmt
ins_affected, ins_err := engine.Insert(LastPan)
fmt.Println(ins_affected)
fmt.Println(ins_err)
}
// count_open_water := open_home_water +open_guest_water
count_real_water := home_real_water + guest_real_water
if count_real_water < 1.75 || count_real_water > 2 {
delete_lastpan2 := new(LastPanEnd)
delete2, _ := engine.Where("schedule_fid=? AND company_cid=? ", schedulefid, cid).Delete(delete_lastpan2)
fmt.Println(delete2)
}
}
return "成功"
}