예제 #1
0
func (this *Updater) Start() {
	ruleFilePath := path.Join(this.rulePath, "update.json")
	dbFilePath := path.Join(this.dbPath, "agent.db")
	tableName := `"update"`

	db, err := util.InitTable(fmt.Sprintf(
		"PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS %s(ip TEXT, port TEXT, source TEXT, level INTEGER)",
		tableName,
	), dbFilePath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	level := util.GetLastLevel(tableName, db) + 1

	e := engine.
		NewQuickEngine(ruleFilePath).
		GetEngine().
		AddPlugin(util.NewAddLevelPlugin(level)).
		SetPipeline(pipeline.NewSqlPipeline(db, tableName))

	var ok bool
	if ok = this.isAgentServerOK(); ok {
		e.AddPlugin(plugin.NewProxyPlugin())
	}
	log.Printf("started %s(isAgentServerOK: %v)\n", ruleFilePath, ok)
	e.Start()
}
예제 #2
0
func main() {
	engine.
		NewQuickEngine("spider.json").
		GetEngine().
		AddPlugin(plugin.NewProxyPlugin()).
		Start()
}
예제 #3
0
func main() {
	if len(os.Args) < 5 {
		log.Printf("lost argument")
		return
	}

	configFilePath, inFilePath, outFilePath, statusFilePath, logFilePath :=
		os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5]

	domains := LoadAndDiff(inFilePath, statusFilePath)
	log.Printf("load %d urls from %s", len(domains), inFilePath)

	outFile, _ := os.OpenFile(outFilePath, os.O_RDWR|os.O_APPEND, 0660)
	defer outFile.Close()
	statusFile, _ := os.OpenFile(statusFilePath, os.O_RDWR|os.O_APPEND, 0660)
	defer statusFile.Close()
	logFile, _ := os.OpenFile(logFilePath, os.O_RDWR|os.O_APPEND, 0660)
	defer logFile.Close()
	log.SetOutput(io.MultiWriter(logFile, os.Stdout))
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	urls := []string{}
	for _, domain := range domains {
		urls = append(urls, "http://bgp.he.net/dns/"+domain)
	}

	e := engine.
		NewQuickEngine(configFilePath).
		SetOutputFile(outFile).
		GetEngine()
	gConfig = e.GetConfig()
	e.SetStartUrls(urls).
		AddPlugin(plugin.NewProxyPlugin()).
		AddPlugin(plugin.NewCookiePlugin(GetCookieFunc)).
		AddPlugin(plugin.NewStatusPlugin(statusFile))
	e.Start()
}