func (this *Fetcher) fetchPage(pageStore PageStore) { defer this.wg.Done() httpClient := lib.HttpClient{} loop: for { select { case taskPack := <-this.taskQueue: destUrl := taskPack.Domain + taskPack.Urlpath html, err := httpClient.Get(destUrl) log.Debugln("goto fetch ", destUrl) done := 0 if err == nil { //report success to scheduler, make a log, save html html, err = httpClient.IconvHtml(html, "utf-8") done = 1 log.Infoln("fetch '" + destUrl + "' done.") err = pageStore.Save(taskPack.Domain, taskPack.Urlpath, string(html)) if err != nil { log.Errorln("fetcher save ", taskPack.Domain, taskPack.Urlpath, " error:", err) } } else { //report fail to scheduler log.Errorln("fetch '"+destUrl+"' failed!", err) } //向scheduler报告任务完成情况 reportUrl := fmt.Sprintf("http://%s%s?task_id=%d&done=%d", this.scheduler_addr, this.scheduler_api["report"], taskPack.TaskId, done) res, err := httpClient.Get(reportUrl) if err != nil { log.Errorln("report ", reportUrl, " failed!") } else { result := types.JsonResult{} err = json.Unmarshal(res, &result) if err != nil || result.Err != 0 { log.Errorln("report ", reportUrl, ", get error response: ", string(res)) } } case <-this.quitChan: //this.quitChan should be closed somewhere log.Infoln("quit fetch page...") break loop } } }