func ListAllSpiders() { l := len(SpiderList.list) logs.Debug("xxxxxxxxxxxxxxxxxxxx: %d", l) for i := 0; i < l; i++ { logs.Debug("%s:%s", SpiderList.list[i].Name, SpiderList.list[i].Address) } }
func downloadPage() { //get task from taskqueue var task = taskqueue.DownloadTask.PopOneTask() logs.Debug("address: %s", task.Spider.Address) resp, ok := DownLoad(&task.Request) if ok != nil { logs.Debug("download error(%s)", ok.Error()) } task.Response = resp taskqueue.PageQueue.PushOneTask(task) logs.Debug("text: %s", task.Response.GetText()) }