func (this *Updater) Start() { ruleFilePath := path.Join(this.rulePath, "update.json") dbFilePath := path.Join(this.dbPath, "agent.db") tableName := `"update"` db, err := util.InitTable(fmt.Sprintf( "PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS %s(ip TEXT, port TEXT, source TEXT, level INTEGER)", tableName, ), dbFilePath) if err != nil { log.Fatal(err) } defer db.Close() level := util.GetLastLevel(tableName, db) + 1 e := engine. NewQuickEngine(ruleFilePath). GetEngine(). AddPlugin(util.NewAddLevelPlugin(level)). SetPipeline(pipeline.NewSqlPipeline(db, tableName)) var ok bool if ok = this.isAgentServerOK(); ok { e.AddPlugin(plugin.NewProxyPlugin()) } log.Printf("started %s(isAgentServerOK: %v)\n", ruleFilePath, ok) e.Start() }
func main() { engine. NewQuickEngine("spider.json"). GetEngine(). AddPlugin(plugin.NewProxyPlugin()). Start() }
func main() { if len(os.Args) < 5 { log.Printf("lost argument") return } configFilePath, inFilePath, outFilePath, statusFilePath, logFilePath := os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5] domains := LoadAndDiff(inFilePath, statusFilePath) log.Printf("load %d urls from %s", len(domains), inFilePath) outFile, _ := os.OpenFile(outFilePath, os.O_RDWR|os.O_APPEND, 0660) defer outFile.Close() statusFile, _ := os.OpenFile(statusFilePath, os.O_RDWR|os.O_APPEND, 0660) defer statusFile.Close() logFile, _ := os.OpenFile(logFilePath, os.O_RDWR|os.O_APPEND, 0660) defer logFile.Close() log.SetOutput(io.MultiWriter(logFile, os.Stdout)) log.SetFlags(log.LstdFlags | log.Lshortfile) urls := []string{} for _, domain := range domains { urls = append(urls, "http://bgp.he.net/dns/"+domain) } e := engine. NewQuickEngine(configFilePath). SetOutputFile(outFile). GetEngine() gConfig = e.GetConfig() e.SetStartUrls(urls). AddPlugin(plugin.NewProxyPlugin()). AddPlugin(plugin.NewCookiePlugin(GetCookieFunc)). AddPlugin(plugin.NewStatusPlugin(statusFile)) e.Start() }