コード例 #1
0
ファイル: json2file.go プロジェクト: yzx226/scrapy
func parseSellerHomePage(host, url string, pageNo int) (string, []string) {
	ll = append(ll, time.Now())
	//fmt.Println(host + url)
	//statusCode, body, err := fasthttp.Get(nil, host+url)
	//if 200 == statusCode && err == nil {
	url = "/i/asynSearch.htm?_ksTS=1466943887482_365&callback=jsonp366&mid=w-14234872766-0&wid=14234872766&path=/search.htm&search=y&pageNo=" + strconv.Itoa(pageNo)
	//url = pickString(body, "J_ShopAsynSearchURL", "value=\"", "\" />")
	fmt.Println("api-->", url)
	if len(url) > 0 {
		ll = append(ll, time.Now())
		statusCode, body, err := fasthttp.Get(nil, host+url)
		//fmt.Println(host+url, len(body))
		if 200 == statusCode && err == nil {
			////rate.taobao.com/batch_query_rate.htm?sellerId=1801924527&itemIds=530915564704,38601200556,42267202365,527986497364,44103007252,42200763824,42422263268,39959198999,521105892359,42434240569,35141045733,523296720589,522132830407,41432065244,525817064353,35499797849,42220550745,41240250690,527066025913,41241875283,43524884406,39280522632,45055179616,38341935049&source=shop
			sellerId := pickString(body, "batch_query_rate.htm", "sellerId=", "&")
			items := pickString(body, "batch_query_rate.htm", "itemIds=", "&")
			seg := ","
			if len(sellerId) == 0 {
				//<textarea id=\"__searchLog\" style=\"display:none\">sInfo=cfb338dc252de504983b9c8bc7989497mall_1.1null13mm_trackidmm_sessionIdsoshewang20160624162118nullnulldefaultSortgridshopid:92686194null1238null3340007498|521219644833|38128004776|44005916004|520815680412|43852003036|35257447217|19219085277|36826027745|16750181194|40893708383|17058029341|13230519823|19332077112|527586744142|36494387102|44358206995|38819051066|44098471350|12901952335|41010377724|36738737167|12493946863|42830490575|42673047135|524784142338|39121491029|19856541019|525701448082|41775383434|36416808624|520132397263|45445033349|13756477016|529694912464|38219850779|41912157546|40582822344|43216130289|16432100352|45212018502|8797141642|40724111979|44362921477|39229001348|529335722507|19588215679|45277933248|9609430242|520509342409|528771660686|532042656548|20572463442|38238878621|524370176919|528947647795|43350508742|521520125592|12913537846|37067711933tmallshophttp
				sellerId = pickString(body, "__searchLog", "gridshopid:", "null")
				items = pickString(body, "__searchLog", "gridshopid", "null", "null", "tmallshophttp")
				seg = "|"
			}
			//fmt.Println(items)

			return sellerId, strings.Split(items, seg)
		}
	}
	//}
	return "", nil
}
コード例 #2
0
ファイル: http_client.go プロジェクト: bogem/nehm
func get(uri string) ([]byte, error) {
	statusCode, body, err := fasthttp.Get(nil, uri)
	if err != nil {
		return nil, err
	}
	if err := handleStatusCode(statusCode); err != nil {
		return nil, err
	}
	return body, nil
}
コード例 #3
0
ファイル: testutils_test.go プロジェクト: phzfi/RIC
// Gets blob from server. package variable port is used as port and localhost as address
func getBlobFromServer(getname string) (blob []byte, err error) {
	statuscode, blob, err := fasthttp.Get(nil, fmt.Sprintf("http://localhost:%d/", port)+getname)
	if statuscode != 200 {
		return nil, errors.New(fmt.Sprintf("Server returned %d", statuscode))
	}
	if err != nil {
		return
	}

	return
}
コード例 #4
0
ファイル: loadimage.go プロジェクト: phzfi/RIC
// Return binary blob of an image from web.
func (img *Image) FromWeb(url string) error {

	statuscode, body, err := fasthttp.Get(nil, url)
	if err != nil {
		return err
	}

	if statuscode != 200 {
		return errors.New(fmt.Sprintf("Couldn't load image. Server returned %d", statuscode))
	}

	return img.FromBlob(body)
}
コード例 #5
0
ファイル: json2file.go プロジェクト: yzx226/scrapy
func json2Filef(url string, save_to string, maxPage string) (int, error) {
	//fmt.Println(url, maxPage)
	for {
		ll = append(ll, time.Now())
		statusCode, body, err := fasthttp.Get(nil, url)
		if err != nil {
			return 0, err
		}
		if 200 != statusCode {
			return 0, fmt.Errorf("!200")
		}

		anti_spider := "smPolicy=tmallrateweb-rate-anti_Spider-checklogin"
		idx_anti_spider := bytes.Index(body, []byte(anti_spider))
		paginator_empty := "\"paginator\":\"\""
		idx_paginator_empty := bytes.Index(body, []byte(paginator_empty))
		if idx_anti_spider >= 0 || idx_paginator_empty >= 0 {
			time.Sleep(5 * time.Second)
			continue
		}

		out := make([]byte, len(body)*2)
		_, bytesWritten, err := iconv.Convert(body, out, "gbk", "utf-8")

		//fmt.Println(bytesRead, bytesWritten, err)
		idx := bytes.IndexByte(out, '(')
		if idx < 0 || bytesWritten < idx+1 {
			return 0, fmt.Errorf("idx error")
		}
		out = out[idx+1 : bytesWritten]
		//fmt.Println(string(out))
		idx_end := bytes.LastIndexByte(out, ')')
		if idx_end < 0 {
			return 0, fmt.Errorf("idx_end<0, )")
		}
		out = out[:idx_end]
		fmt.Println(save_to)
		ioutil.WriteFile(save_to, out, 0666)

		time.Sleep(1 * time.Second)
		if len(maxPage) > 0 {
			return strconv.Atoi(pickString(body, maxPage, ":", ","))
		} else {
			return 0, nil
		}
	}
}
コード例 #6
0
ファイル: engine.go プロジェクト: yzx226/scrapy
func fetchData(url string) ([]byte, bool) {

	for i := 0; ; i++ {
		if i > 3 {
			return nil, false
		}
		statusCode, body, err := fasthttp.Get(nil, url)

		if statusCode != 200 || err != nil {
			fmt.Println(statusCode, err)
			<-time.After(10 * time.Second)
			break
		}

		return body, true
	}
	return nil, false
}