コード例 #1
0
ファイル: scheduler.go プロジェクト: zhaozhi406/crawler
func (this *Scheduler) reportTaskHandler(w http.ResponseWriter, req *http.Request) {
	requiredParams := map[string]string{"task_id": "int", "done": "int"}
	_, err := utils.CheckHttpParams(req, requiredParams)
	result := types.JsonResult{}
	if err != nil {
		log.Errorln(err)
		result.Err = ErrInputError
		result.Msg = err.Error()
		utils.OutputJsonResult(w, result)
		return
	}

	taskId := req.Form.Get("task_id")
	intTaskId, _ := strconv.Atoi(taskId)
	task := types.CrawlTask{Id: int32(intTaskId)}
	tasks := []types.CrawlTask{task}
	done := req.Form.Get("done")
	var status dao.TaskStatus
	if done == "1" {
		status = dao.TASK_FINISH
	} else {
		status = dao.TASK_FAILED
	}
	_, err = this.taskDao.SetTasksStatus(tasks, status)
	if err != nil {
		msg := fmt.Sprintf("set task %s status to %d, error: %v", taskId, status, err)
		log.Errorln(msg)
		result.Err = ErrDbError
		result.Msg = msg

	} else {
		log.Errorln("set task ", taskId, " status to ", status, " finished.")
		result.Err = ErrOk
	}
	utils.OutputJsonResult(w, result)
}
コード例 #2
0
ファイル: fetcher.go プロジェクト: zhaozhi406/crawler
func (this *Fetcher) pushTasksHandler(w http.ResponseWriter, req *http.Request) {
	log.Debugln("get request: ", req.RemoteAddr, req.URL)
	req.ParseForm()
	tasksJson := req.Form.Get("tasks")
	taskPacks := []types.TaskPack{}
	var err error = nil
	var result = types.JsonResult{}
	if tasksJson != "" {
		err = json.Unmarshal([]byte(tasksJson), &taskPacks)
		if err != nil {
			msg := "Unmarshal task packs error: " + err.Error()
			log.Errorln(msg)
			result.Err = ErrDataError
			result.Msg = msg
		} else {
			//添加任务到队列
			//最多只允许执行1秒钟
			timerChan := time.After(1 * time.Second)
			cnt := 0
			for _, pack := range taskPacks {
				select {
				case this.taskQueue <- pack:
					cnt++
				case <-timerChan:
					break
				}
			}
			result.Err = ErrOk
			result.Data = taskPacks[:cnt] //将成功进入队列的任务返回
		}
	} else {
		msg := "missing `tasks` key or has no content in the POST request."
		log.Warnln(msg)
		result.Err = ErrInputError
		result.Msg = msg
	}
	utils.OutputJsonResult(w, result)
}