コード例 #1
0
ファイル: distributer.go プロジェクト: ROOT005/ants-go
// if cookiejar > 0 means it require cookie context ,so we should send it to where it come from
// else distribute it by order
func (this *Distributer) Distribute(request *http.Request) {
	if request.CookieJar > 0 {
		return
	} else {
		nodeList := this.Cluster.GetAllNode()
		if this.LastIndex >= len(nodeList) {
			this.LastIndex = 0
		}
		nodeName := nodeList[this.LastIndex].Name
		request.NodeName = nodeName
		this.LastIndex += 1
	}
}
コード例 #2
0
ファイル: downloader.go プロジェクト: ROOT005/ants-go
// download it and push in goroutine
func (this *Downloader) downloadAndPush(request *http.Request) {
	log.Println(request.SpiderName, "depth:", request.Depth, "download url:", request.GoRequest.URL.String())
	client := this.getClient(request)
	if request.Proxy != "" {
		client.SetProxy(request.Proxy)
	}
	response, err := client.GoClient.Do(request.GoRequest)
	if err != nil {
		log.Println(err)
		if request.Retry < 3 {
			log.Println(request.SpiderName, "error time", request.Retry, "return to quene:", request.GoRequest.URL.String())
			request.Retry += 1
			this.RequestQuene.Push(request)
			return
		} else {
			log.Println(request.SpiderName, "error time", request.Retry, "drop:", request.GoRequest.URL.String())
		}
	}
	client.ClearProxy()
	Response := http.NewResponse(response, request, request.SpiderName, request.ParserName, request.NodeName)
	this.ResponseQuene.Push(Response)
}