func (this *Updater) Start() {
ruleFilePath := path.Join(this.rulePath, "update.json")
dbFilePath := path.Join(this.dbPath, "agent.db")
tableName := `"update"`
db, err := util.InitTable(fmt.Sprintf(
"PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS %s(ip TEXT, port TEXT, source TEXT, level INTEGER)",
tableName,
), dbFilePath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
level := util.GetLastLevel(tableName, db) + 1
e := engine.
NewQuickEngine(ruleFilePath).
GetEngine().
AddPlugin(util.NewAddLevelPlugin(level)).
SetPipeline(pipeline.NewSqlPipeline(db, tableName))
var ok bool
if ok = this.isAgentServerOK(); ok {
e.AddPlugin(plugin.NewProxyPlugin())
}
log.Printf("started %s(isAgentServerOK: %v)\n", ruleFilePath, ok)
e.Start()
}
func main() {
engine.
NewQuickEngine("spider.json").
GetEngine().
AddPlugin(plugin.NewProxyPlugin()).
Start()
}
func (this *Validator) Start() {
ruleFilePath := path.Join(this.rulePath, "validate.json")
dbFilePath := path.Join(this.dbPath, "agent.db")
updateTableName := `"update"`
validateTableName := `"validate"`
db, err := util.InitTable(fmt.Sprintf(
"PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS %s(ip TEXT, port TEXT, domain TEXT, level INTEGER)",
validateTableName,
), dbFilePath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
level := util.GetLastLevel(validateTableName, db) + 1
qe := engine.NewQuickEngine(ruleFilePath)
reqs := this.genRequests(qe.GetQuickEngineConfig().StartUrls, updateTableName, level, db)
qe.
GetEngine().
SetStartRequests(reqs).
AddPlugin(util.NewModifyResponsePlugin()).
AddPlugin(util.NewAddLevelPlugin(level)).
SetPipeline(pipeline.NewSqlPipeline(db, validateTableName)).
Start()
}
func main() {
if len(os.Args) < 5 {
log.Printf("lost argument")
return
}
configFilePath, inFilePath, outFilePath, statusFilePath, logFilePath :=
os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5]
domains := LoadAndDiff(inFilePath, statusFilePath)
log.Printf("load %d urls from %s", len(domains), inFilePath)
outFile, _ := os.OpenFile(outFilePath, os.O_RDWR|os.O_APPEND, 0660)
defer outFile.Close()
statusFile, _ := os.OpenFile(statusFilePath, os.O_RDWR|os.O_APPEND, 0660)
defer statusFile.Close()
logFile, _ := os.OpenFile(logFilePath, os.O_RDWR|os.O_APPEND, 0660)
defer logFile.Close()
log.SetOutput(io.MultiWriter(logFile, os.Stdout))
log.SetFlags(log.LstdFlags | log.Lshortfile)
urls := []string{}
for _, domain := range domains {
urls = append(urls, "http://bgp.he.net/dns/"+domain)
}
e := engine.
NewQuickEngine(configFilePath).
SetOutputFile(outFile).
GetEngine()
gConfig = e.GetConfig()
e.SetStartUrls(urls).
AddPlugin(plugin.NewProxyPlugin()).
AddPlugin(plugin.NewCookiePlugin(GetCookieFunc)).
AddPlugin(plugin.NewStatusPlugin(statusFile))
e.Start()
}
func main() {
engine.NewQuickEngine("crawl_baidubaike_with_quickengine.json").Start()
}