func ListAllSpiders() {
l := len(SpiderList.list)
logs.Debug("xxxxxxxxxxxxxxxxxxxx: %d", l)
for i := 0; i < l; i++ {
logs.Debug("%s:%s", SpiderList.list[i].Name, SpiderList.list[i].Address)
}
}
func downloadPage() {
//get task from taskqueue
var task = taskqueue.DownloadTask.PopOneTask()
logs.Debug("address: %s", task.Spider.Address)
resp, ok := DownLoad(&task.Request)
if ok != nil {
logs.Debug("download error(%s)", ok.Error())
}
task.Response = resp
taskqueue.PageQueue.PushOneTask(task)
logs.Debug("text: %s", task.Response.GetText())
}