// if cookiejar > 0 means it require cookie context ,so we should send it to where it come from
// else distribute it by order
func (this *Distributer) Distribute(request *http.Request) {
if request.CookieJar > 0 {
return
} else {
nodeList := this.Cluster.GetAllNode()
if this.LastIndex >= len(nodeList) {
this.LastIndex = 0
}
nodeName := nodeList[this.LastIndex].Name
request.NodeName = nodeName
this.LastIndex += 1
}
}
// download it and push in goroutine
func (this *Downloader) downloadAndPush(request *http.Request) {
log.Println(request.SpiderName, "depth:", request.Depth, "download url:", request.GoRequest.URL.String())
client := this.getClient(request)
if request.Proxy != "" {
client.SetProxy(request.Proxy)
}
response, err := client.GoClient.Do(request.GoRequest)
if err != nil {
log.Println(err)
if request.Retry < 3 {
log.Println(request.SpiderName, "error time", request.Retry, "return to quene:", request.GoRequest.URL.String())
request.Retry += 1
this.RequestQuene.Push(request)
return
} else {
log.Println(request.SpiderName, "error time", request.Retry, "drop:", request.GoRequest.URL.String())
}
}
client.ClearProxy()
Response := http.NewResponse(response, request, request.SpiderName, request.ParserName, request.NodeName)
this.ResponseQuene.Push(Response)
}