// FetchRzrqSumData 抓取数据 func FetchRzrqSumData(page int) ([]byte, error) { st := gos.NowInLocation().Unix() / 30 formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=%s&st=0&sr=1&p=%d&ps=50&js=var%%20ruOtumOo={pages:(pc),data:[(x)]}&rt=%d" url := fmt.Sprintf(formt, "SHSZHSSUM", page, st) body, err := wget.GetBody(url) if err != nil { return nil, gos.DoError(err) } return body[bytes.Index(body, []byte("[")) : len(body)-1], nil }
// FetchPingGuData 抓取行业评估数据 func FetchPingGuData(code string) (*PingGuStruct, error) { // http://stockData.stock.hexun.com/600028.shtml now := gos.NowInLocation() formt := "http://pinggu.stock.hexun.com/DataProvider/StockFinanceNewFlash.ashx?&code=%s&s=%d" body, err := wget.GetBody(fmt.Sprintf(formt, code, now.Unix())) if err != nil { return nil, gos.DoError(err) } v := &PingGuStruct{} err = xml.Unmarshal(body, v) if err != nil { return nil, gos.DoError(err) } quarter := util.StockQuarter() for _, item := range v.Content { item.Date = quarter switch item.Name { case "综合能力": item.Itype = 1 case "盈利能力": item.Itype = 2 case "偿债能力": item.Itype = 3 case "成长能力": item.Itype = 4 case "资产经营": item.Itype = 5 case "市场表现": item.Itype = 6 case "投资收益": item.Itype = 7 } } return v, nil }
// http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=0.6341747129336 // curl "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=0.6341747129336" -H "Pragma: no-cache" -H "Accept-Encoding: gzip, deflate, sdch" -H "Accept-Language: en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36" -H "Accept: */*" -H "Referer: http://data.eastmoney.com/bkzj/hgt.html" -H "X-Requested-With: ShockwaveFlash/18.0.0.232" -H "Cookie: HAList=f-0-000003-B"%"u80A1"%"u6307"%"u6570; pgv_pvi=3511567662; pgv_info=ssi=s7100762258" -H "Connection: keep-alive" -H "Cache-Control: no-cache" --compressed // RzrqSumData 抓取数据 func GetHgtAmount() ([]*HgtAmount, error) { r := rand.New(rand.NewSource(99)) formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=SHT&sty=SHTTMYE&rt=%v" body, err := wget.GetBody(fmt.Sprintf(formt, r.Float64())) if err != nil { return nil, gos.DoError(err) } now := gos.NowInLocation() arr := bytes.Split(body, []byte("\r\n")) var tmp [][]byte result := make([]*HgtAmount, len(arr)) var amountA float64 var amountB float64 var date time.Time for i, v := range arr { tmp = bytes.Split(v, gos.B_SEMICOLON) amountA, err = util.Str2Float64(string(tmp[1])) if err != nil { amountA = 0 } amountB, err = util.Str2Float64(string(tmp[2])) if err != nil { amountA = 0 } date, err = time.ParseInLocation("2006/1/2 15:04", fmt.Sprint(now.Format("2006/1/2"), " ", string(tmp[0])), gos.GetSite().Location) if err != nil { date = now } result[i] = &HgtAmount{ Date: date, AmountA: amountA, AmountB: amountB, } } return result, nil }
// fetchAndFillYyb 抓取数据 func FetchGgcg(page int) (*jsonGgcgData, error) { fmt.Println("fetch page ", page) // http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=2&ps=50&js=var%20XvAVhGPE={%22data%22:[(x)],%22pages%22:%22(pc)%22,%22update%22:%22(ud)%22}&rt=48257541 formt := `http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=GG&sty=GGMX&p=%d&ps=%d&js=var%%20ftzqaLVS={pages:(pc),data:[(x)]}&rt=%d` pageLimit := 100 body, err := wget.GetBody(fmt.Sprintf(formt, page, pageLimit, time.Now().Unix())) if err != nil { return nil, err } src := body[13:] src = bytes.Replace(src, []byte("pages:"), []byte(`"pages":`), 1) src = bytes.Replace(src, []byte(",data:"), []byte(`,"data":`), 1) v := &jsonGgcgData{} err = json.Unmarshal(src, v) if err != nil { return nil, err } return v, nil }
// FetchRzrqStockData 抓取数据 func FetchRzrqStockData(code string, page int) ([]byte, error) { st := time.Now().Unix() / 30 // var% OKPJKmpr={pages:10,data: // http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=1&code=600718&st=0&sr=1&p=5&ps=50&js=var%20OKPJKmpr={pages:(pc),data:[(x)]}&rt=48027423 // http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=2&code=002161&st=0&sr=1&p=5&ps=50&js=var%20QOwpoBrj={pages:(pc),data:[(x)]}&rt=48154207 // http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=2&code=300001&st=0&sr=1&p=5&ps=50&js=var%20laheCxDp={pages:(pc),data:[(x)]}&rt=48154224 formt := "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=FD&sty=MTE&mkt=%d&code=%s&st=0&sr=1&p=%d&ps=50&js=var%%20OKPJKmpr={pages:(pc),data:[(x)]}&rt=%d" mkt := 2 switch code[0:1] { case "6": mkt = 1 } url := fmt.Sprintf(formt, mkt, code, page, st) body, err := wget.GetBody(url) if err != nil { return nil, gos.DoError(err) } // var OKPJKmpr={pages:0,data:[{stats:false}]} return body[bytes.Index(body, []byte("[")) : len(body)-1], nil }
func fetchAndFillFundRank(page int, quarter string) (int, error) { // http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=50&date=2015-09-30&orgType=&stateType=null&titType=null&page=2&callback=hxbase_json7 formt := "http://stockdata.stock.hexun.com/jgcc/data/outdata/orgrank.ashx?count=%s&date=%s&orgType=&stateType=null&titType=null&page=%d&callback=hxbase_json7" pageLimit := 100 body, err := wget.GetBody(fmt.Sprintf(formt, pageLimit, quarter, page)) if err != nil { return -1, err } exists := db.NewExistsBuilder("funds") ist := db.NewInsertBuilder("funds") query := db.NewQueryBuilder("funds") var fund *Fund var fundRank *FundRank var row db.DataRow // hxbase_json7( str, err := iconv.ConvertString(string(body), "gb2312", "utf-8") if err != nil { return -1, err } src := []byte(str) src = src[13 : len(src)-1] src = bytes.Replace(src, []byte(":'"), []byte(`":"`), -1) src = bytes.Replace(src, []byte("',"), []byte(`","`), -1) src = bytes.Replace(src, []byte("'}"), []byte(`"}`), -1) src = bytes.Replace(src, []byte("{"), []byte(`{"`), -1) src = bytes.Replace(src, []byte("sum:"), []byte(`sum":`), 1) src = bytes.Replace(src, []byte("list:"), []byte(`"list":`), 1) v := &JsonFund{} err = json.Unmarshal(src, v) if err != nil { fmt.Println(string(src)) return -1, err } // {RankTd:'51',OrgName:'法国巴黎银行',OrgNameLink:'o-QF000031.shtml',OrgType:'QFII',ShareHoldingNum:'3',ShareHoldingNumLink:'otherDetail.aspx?OrgNo=QF000031',TotalHoldings:'48,388.00',TotalMarketValue:'1,100,735.00',OrgAlt:'法国巴黎银行'} for _, item := range v.List { fund = &Fund{Code: item["OrgNameLink"], Name: string(item["OrgName"]), TypeID: FundType(item["OrgType"])} fund.Code = fund.Code[2 : len(fund.Code)-6] fmt.Println("----") if !exists.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).Exists() { fmt.Println("insert", fund) ist.Table("funds").Insert(fund) } row, _ = query.Table("funds").Where("code=? and type_id=?", fund.Code, fund.TypeID).QueryOne() if row.Empty() { return -1, fmt.Errorf("code %s not found", fund.Code) } fundRank = &FundRank{ FundID: row.GetInt64("id"), Date: quarter, Rank: int(util.ParseMoney(item["RankTd"])), Count: int(util.ParseMoney(item["ShareHoldingNum"])), MH: int64(util.ParseMoney(item["TotalHoldings"])), MV: int64(util.ParseMoney(item["TotalMarketValue"])), } if !exists.Table("fund_rank").Where("fund_id=? and date=?", fundRank.FundID, fundRank.Date).Exists() { fmt.Println("insert", fundRank) ist.Table("fund_rank").Insert(fundRank) } } return int(math.Ceil(float64(v.Sum / pageLimit))), nil }
// fetchAndFillYyb 抓取数据 func fetchAndFillYyb(page int) (int, error) { // http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=2&ps=50&js=var%20XvAVhGPE={%22data%22:[(x)],%22pages%22:%22(pc)%22,%22update%22:%22(ud)%22}&rt=48257541 formt := `http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx?type=LHB&sty=YYTJ&stat=6&sr=0&st=1&p=%d&ps=%d&js=var%%20XvAVhGPE={"data":[(x)],"pages":"(pc)","update":"(ud)"}&rt=%d` pageLimit := 100 body, err := wget.GetBody(fmt.Sprintf(formt, page, pageLimit, time.Now().Unix())) if err != nil { return -1, err } exists := db.NewExistsBuilder("yyb") ist := db.NewInsertBuilder("yyb") upd := db.NewUpdateBuilder("yyb") query := db.NewQueryBuilder("yyb") var yyb *yybStruct var yybLub *yybLhbStruct var arr []string var row db.DataRow src := body[bytes.Index(body, []byte("={"))+1:] v := &jsonYybData{} err = json.Unmarshal(src, v) if err != nil { return -1, err } // "80136686,2,216,135,七喜控股|协鑫集成|恒信移动,2991067145.735,广东,80000073,440000,105,6089657316.805,3098590171.07,华泰证券股份有限公司广州天河东路证券营业部,002027.SZ|002506.SZ|300081.SZ" for i, item := range v.Data { arr = strings.Split(item, ",") yyb = &yybStruct{Code: arr[0], Name: arr[12], Area: arr[6]} if !exists.Table("yyb").Where("code=?", yyb.Code).Exists() { gos.Log.Info("insert", yyb.Code) ist.Table("yyb").Insert(yyb) } row, _ = query.Table("yyb").Where("code=?", yyb.Code).QueryOne() if row.Empty() { return -1, fmt.Errorf("code %s not found", yyb.Code) } yybLub = &yybLhbStruct{ YybID: row.GetInt64("id"), Updated: v.Update, Rank: pageLimit*(page-1) + i + 1, Amount: util.ParseMoney(arr[10]), Buy: util.ParseMoney(arr[11]), Sell: util.ParseMoney(arr[5]), Num: int(util.ParseMoney(arr[2])), NumBuy: int(util.ParseMoney(arr[3])), NumSell: int(util.ParseMoney(arr[9])), } if exists.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Exists() { gos.Log.Info("update", yyb.Code, yybLub.YybID) upd.Table("yyb_lhb").Where("yyb_id=?", yybLub.YybID).Update(yybLub) } else { gos.Log.Info("insert", yyb.Code, yybLub.YybID) ist.Table("yyb_lhb").Insert(yybLub) } } return int(util.ParseMoney(v.Pages)), nil }