func ChkPuppeteerConf(puppeteerConf *PuppeteerConf) bool {
if nil == puppeteerConf {
return false
}
os.MkdirAll(puppeteerConf.PoolDir, ppioutil.DIR_MASK)
os.MkdirAll(puppeteerConf.QueueDir, ppioutil.DIR_MASK)
initDir := ppqueue.GetJobInitDir(puppeteerConf.QueueDir)
runDir := ppqueue.GetJobRunDir(puppeteerConf.QueueDir)
waitDir := ppqueue.GetJobWaitDir(puppeteerConf.QueueDir)
os.MkdirAll(initDir, ppioutil.DIR_MASK)
os.MkdirAll(runDir, ppioutil.DIR_MASK)
os.MkdirAll(waitDir, ppioutil.DIR_MASK)
if !ppioutil.IsDirExists(puppeteerConf.PoolDir) {
return false
}
if !ppioutil.IsDirExists(puppeteerConf.QueueDir) {
return false
}
_, err := os.Stat(puppeteerConf.PhantomJSBin)
if nil != err {
return false
}
return true
}
func JobMaster(queueChannel chan string, scoreboard *Scoreboard) {
scoreboard.IncrProcCnt()
procCnt := scoreboard.GetProcCnt()
scoreboard.Lock.RLock()
queueDir := scoreboard.Conf.QueueDir
maxProc := scoreboard.Conf.MaxProc
scoreboard.Lock.RUnlock()
log.Printf("job master starts")
for idx := procCnt; idx < maxProc; idx++ {
go JobSlave(queueChannel, scoreboard)
}
for {
if scoreboard.IsTerminated() {
break
}
waitDir := ppqueue.GetJobWaitDir(queueDir)
if dirHandle, err := os.Open(waitDir); nil == err {
for {
fileList, err := dirHandle.Readdir(1)
if nil != err || 0 >= len(fileList) {
break
}
queueFile := waitDir + string(os.PathSeparator) + fileList[0].Name()
queueChannel <- queueFile
}
dirHandle.Close()
} else {
log.Printf("open queue dir error - %s", err)
}
time.Sleep(time.Second)
}
close(queueChannel)
scoreboard.DecrProcCnt()
log.Printf("master stops\n")
}
