Beispiel #1
0
func ChkPuppeteerConf(puppeteerConf *PuppeteerConf) bool {
	if nil == puppeteerConf {
		return false
	}

	os.MkdirAll(puppeteerConf.PoolDir, ppioutil.DIR_MASK)
	os.MkdirAll(puppeteerConf.QueueDir, ppioutil.DIR_MASK)
	initDir := ppqueue.GetJobInitDir(puppeteerConf.QueueDir)
	runDir := ppqueue.GetJobRunDir(puppeteerConf.QueueDir)
	waitDir := ppqueue.GetJobWaitDir(puppeteerConf.QueueDir)
	os.MkdirAll(initDir, ppioutil.DIR_MASK)
	os.MkdirAll(runDir, ppioutil.DIR_MASK)
	os.MkdirAll(waitDir, ppioutil.DIR_MASK)

	if !ppioutil.IsDirExists(puppeteerConf.PoolDir) {
		return false
	}

	if !ppioutil.IsDirExists(puppeteerConf.QueueDir) {
		return false
	}

	_, err := os.Stat(puppeteerConf.PhantomJSBin)
	if nil != err {
		return false
	}

	return true
}
Beispiel #2
0
func JobSlave(queueChannel chan string, scoreboard *Scoreboard) {
	if !scoreboard.IncrProcCnt() {
		return
	}

	scoreboard.Lock.RLock()
	queueDir := scoreboard.Conf.QueueDir
	phantomJSBin := scoreboard.Conf.PhantomJSBin
	jsPath := scoreboard.Conf.JS
	expire := scoreboard.Conf.Expire
	scoreboard.Lock.RUnlock()

	log.Printf("job slave starts")
	t := time.NewTimer(time.Second)
	for {
		if scoreboard.IsTerminated() {
			break
		}

		select {
		case queueFile, queueValid := <-queueChannel:
			if !queueValid {
				break
			}

			if sepIdx := strings.LastIndex(queueFile, string(os.PathSeparator)); -1 != sepIdx {
				runDir := ppqueue.GetJobRunDir(queueDir)
				queueFileName := string(queueFile[sepIdx+1:])
				runFile := runDir + string(os.PathSeparator) + queueFileName

				if err := os.Rename(queueFile, runFile); nil == err {
					timestamp := time.Now().Unix()
					if jobInfo := ppqueue.ReadJob(runFile); nil != jobInfo {
						if fileStat, statErr := os.Stat(jobInfo[ppqueue.TARGET_FILE]); (nil != statErr && os.IsNotExist(statErr)) || (nil == statErr && expire < (timestamp-fileStat.ModTime().Unix())) {
							log.Printf("process job %s for %s\n", runFile, jobInfo[ppqueue.TARGET_FILE])
							log.Printf("process job %s begins\n", runFile)
							cmd := exec.Command(phantomJSBin, jsPath, jobInfo[ppqueue.URL], jobInfo[ppqueue.TARGET_FILE], jobInfo[ppqueue.LOG_FILE], jobInfo[ppqueue.USER_AGENT])
							log.Printf("process job %s ends\n", runFile)
							if err := cmd.Run(); nil != err {
								log.Printf("process job err - %s\n", err.Error())
							}
						}
					}

					os.Remove(runFile)
				}
			}
		case <-t.C:
		}
		time.Sleep(time.Second)
	}

	scoreboard.DecrProcCnt()
	log.Printf("slave stops\n")
}