func ChkPuppeteerConf(puppeteerConf *PuppeteerConf) bool {
if nil == puppeteerConf {
return false
}
os.MkdirAll(puppeteerConf.PoolDir, ppioutil.DIR_MASK)
os.MkdirAll(puppeteerConf.QueueDir, ppioutil.DIR_MASK)
initDir := ppqueue.GetJobInitDir(puppeteerConf.QueueDir)
runDir := ppqueue.GetJobRunDir(puppeteerConf.QueueDir)
waitDir := ppqueue.GetJobWaitDir(puppeteerConf.QueueDir)
os.MkdirAll(initDir, ppioutil.DIR_MASK)
os.MkdirAll(runDir, ppioutil.DIR_MASK)
os.MkdirAll(waitDir, ppioutil.DIR_MASK)
if !ppioutil.IsDirExists(puppeteerConf.PoolDir) {
return false
}
if !ppioutil.IsDirExists(puppeteerConf.QueueDir) {
return false
}
_, err := os.Stat(puppeteerConf.PhantomJSBin)
if nil != err {
return false
}
return true
}
func JobSlave(queueChannel chan string, scoreboard *Scoreboard) {
if !scoreboard.IncrProcCnt() {
return
}
scoreboard.Lock.RLock()
queueDir := scoreboard.Conf.QueueDir
phantomJSBin := scoreboard.Conf.PhantomJSBin
jsPath := scoreboard.Conf.JS
expire := scoreboard.Conf.Expire
scoreboard.Lock.RUnlock()
log.Printf("job slave starts")
t := time.NewTimer(time.Second)
for {
if scoreboard.IsTerminated() {
break
}
select {
case queueFile, queueValid := <-queueChannel:
if !queueValid {
break
}
if sepIdx := strings.LastIndex(queueFile, string(os.PathSeparator)); -1 != sepIdx {
runDir := ppqueue.GetJobRunDir(queueDir)
queueFileName := string(queueFile[sepIdx+1:])
runFile := runDir + string(os.PathSeparator) + queueFileName
if err := os.Rename(queueFile, runFile); nil == err {
timestamp := time.Now().Unix()
if jobInfo := ppqueue.ReadJob(runFile); nil != jobInfo {
if fileStat, statErr := os.Stat(jobInfo[ppqueue.TARGET_FILE]); (nil != statErr && os.IsNotExist(statErr)) || (nil == statErr && expire < (timestamp-fileStat.ModTime().Unix())) {
log.Printf("process job %s for %s\n", runFile, jobInfo[ppqueue.TARGET_FILE])
log.Printf("process job %s begins\n", runFile)
cmd := exec.Command(phantomJSBin, jsPath, jobInfo[ppqueue.URL], jobInfo[ppqueue.TARGET_FILE], jobInfo[ppqueue.LOG_FILE], jobInfo[ppqueue.USER_AGENT])
log.Printf("process job %s ends\n", runFile)
if err := cmd.Run(); nil != err {
log.Printf("process job err - %s\n", err.Error())
}
}
}
os.Remove(runFile)
}
}
case <-t.C:
}
time.Sleep(time.Second)
}
scoreboard.DecrProcCnt()
log.Printf("slave stops\n")
}