func parseSellerHomePage(host, url string, pageNo int) (string, []string) {
ll = append(ll, time.Now())
//fmt.Println(host + url)
//statusCode, body, err := fasthttp.Get(nil, host+url)
//if 200 == statusCode && err == nil {
url = "/i/asynSearch.htm?_ksTS=1466943887482_365&callback=jsonp366&mid=w-14234872766-0&wid=14234872766&path=/search.htm&search=y&pageNo=" + strconv.Itoa(pageNo)
//url = pickString(body, "J_ShopAsynSearchURL", "value=\"", "\" />")
fmt.Println("api-->", url)
if len(url) > 0 {
ll = append(ll, time.Now())
statusCode, body, err := fasthttp.Get(nil, host+url)
//fmt.Println(host+url, len(body))
if 200 == statusCode && err == nil {
////rate.taobao.com/batch_query_rate.htm?sellerId=1801924527&itemIds=530915564704,38601200556,42267202365,527986497364,44103007252,42200763824,42422263268,39959198999,521105892359,42434240569,35141045733,523296720589,522132830407,41432065244,525817064353,35499797849,42220550745,41240250690,527066025913,41241875283,43524884406,39280522632,45055179616,38341935049&source=shop
sellerId := pickString(body, "batch_query_rate.htm", "sellerId=", "&")
items := pickString(body, "batch_query_rate.htm", "itemIds=", "&")
seg := ","
if len(sellerId) == 0 {
//<textarea id=\"__searchLog\" style=\"display:none\">sInfo=cfb338dc252de504983b9c8bc7989497mall_1.1null13mm_trackidmm_sessionIdsoshewang20160624162118nullnulldefaultSortgridshopid:92686194null1238null3340007498|521219644833|38128004776|44005916004|520815680412|43852003036|35257447217|19219085277|36826027745|16750181194|40893708383|17058029341|13230519823|19332077112|527586744142|36494387102|44358206995|38819051066|44098471350|12901952335|41010377724|36738737167|12493946863|42830490575|42673047135|524784142338|39121491029|19856541019|525701448082|41775383434|36416808624|520132397263|45445033349|13756477016|529694912464|38219850779|41912157546|40582822344|43216130289|16432100352|45212018502|8797141642|40724111979|44362921477|39229001348|529335722507|19588215679|45277933248|9609430242|520509342409|528771660686|532042656548|20572463442|38238878621|524370176919|528947647795|43350508742|521520125592|12913537846|37067711933tmallshophttp
sellerId = pickString(body, "__searchLog", "gridshopid:", "null")
items = pickString(body, "__searchLog", "gridshopid", "null", "null", "tmallshophttp")
seg = "|"
}
//fmt.Println(items)
return sellerId, strings.Split(items, seg)
}
}
//}
return "", nil
}
func get(uri string) ([]byte, error) {
statusCode, body, err := fasthttp.Get(nil, uri)
if err != nil {
return nil, err
}
if err := handleStatusCode(statusCode); err != nil {
return nil, err
}
return body, nil
}
// Gets blob from server. package variable port is used as port and localhost as address
func getBlobFromServer(getname string) (blob []byte, err error) {
statuscode, blob, err := fasthttp.Get(nil, fmt.Sprintf("http://localhost:%d/", port)+getname)
if statuscode != 200 {
return nil, errors.New(fmt.Sprintf("Server returned %d", statuscode))
}
if err != nil {
return
}
return
}
// Return binary blob of an image from web.
func (img *Image) FromWeb(url string) error {
statuscode, body, err := fasthttp.Get(nil, url)
if err != nil {
return err
}
if statuscode != 200 {
return errors.New(fmt.Sprintf("Couldn't load image. Server returned %d", statuscode))
}
return img.FromBlob(body)
}
func json2Filef(url string, save_to string, maxPage string) (int, error) {
//fmt.Println(url, maxPage)
for {
ll = append(ll, time.Now())
statusCode, body, err := fasthttp.Get(nil, url)
if err != nil {
return 0, err
}
if 200 != statusCode {
return 0, fmt.Errorf("!200")
}
anti_spider := "smPolicy=tmallrateweb-rate-anti_Spider-checklogin"
idx_anti_spider := bytes.Index(body, []byte(anti_spider))
paginator_empty := "\"paginator\":\"\""
idx_paginator_empty := bytes.Index(body, []byte(paginator_empty))
if idx_anti_spider >= 0 || idx_paginator_empty >= 0 {
time.Sleep(5 * time.Second)
continue
}
out := make([]byte, len(body)*2)
_, bytesWritten, err := iconv.Convert(body, out, "gbk", "utf-8")
//fmt.Println(bytesRead, bytesWritten, err)
idx := bytes.IndexByte(out, '(')
if idx < 0 || bytesWritten < idx+1 {
return 0, fmt.Errorf("idx error")
}
out = out[idx+1 : bytesWritten]
//fmt.Println(string(out))
idx_end := bytes.LastIndexByte(out, ')')
if idx_end < 0 {
return 0, fmt.Errorf("idx_end<0, )")
}
out = out[:idx_end]
fmt.Println(save_to)
ioutil.WriteFile(save_to, out, 0666)
time.Sleep(1 * time.Second)
if len(maxPage) > 0 {
return strconv.Atoi(pickString(body, maxPage, ":", ","))
} else {
return 0, nil
}
}
}
func fetchData(url string) ([]byte, bool) {
for i := 0; ; i++ {
if i > 3 {
return nil, false
}
statusCode, body, err := fasthttp.Get(nil, url)
if statusCode != 200 || err != nil {
fmt.Println(statusCode, err)
<-time.After(10 * time.Second)
break
}
return body, true
}
return nil, false
}