Esempio n. 1
0
func ListAllSpiders() {
	l := len(SpiderList.list)
	logs.Debug("xxxxxxxxxxxxxxxxxxxx: %d", l)
	for i := 0; i < l; i++ {
		logs.Debug("%s:%s", SpiderList.list[i].Name, SpiderList.list[i].Address)
	}
}
Esempio n. 2
0
func downloadPage() {
	//get task from taskqueue
	var task = taskqueue.DownloadTask.PopOneTask()
	logs.Debug("address: %s", task.Spider.Address)

	resp, ok := DownLoad(&task.Request)
	if ok != nil {
		logs.Debug("download error(%s)", ok.Error())
	}
	task.Response = resp
	taskqueue.PageQueue.PushOneTask(task)
	logs.Debug("text: %s", task.Response.GetText())
}