func main() { //var wg sync.WaitGroup //读取启动参数 var configFile string flag.StringVar(&configFile, "conf", "config.ini", "configure file full path") flag.Parse() fmt.Printf("Start AccipitrSpiderEngine...\n") //启动日志系统 logger, err := log4jzl.New("AccipitrSpiderEngine") if err != nil { fmt.Printf("[ERROR] Create logger Error: %v\n", err) //return } //读取配置文件 configure, err := u.NewConfigure(configFile) if err != nil { fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err) return } //初始化数据库适配器 /* dbAdaptor, err := u.NewDBAdaptor(configure, logger) if err != nil { fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err) return } defer dbAdaptor.Release() */ //初始化KVDB数据库 DBname, _ := configure.GetKVDB() var kvdb *u.PandionKV if !Exist(fmt.Sprintf("./DB/%v.idx", DBname)) { kvdb = u.NewPandionKV(DBname, logger) } else { kvdb = u.NewPandionKVWithFile(DBname, logger) } router := &Router{Kvdb: kvdb} //初始化自定义的插件 ps := make(map[string]u.PlugInterface) plug := Plugs.NewYXQ("yingxiaoqun", logger, nil) // dbAdaptor) plugyo := Plugs.NewRadialeng("radialeng", logger, nil) maudio := Plugs.NewMAudio("maudio", logger, nil) jblpro := Plugs.NewJblpro("jblpor", logger, nil) ikmultimedia := Plugs.NewIkmultimedia("ikmultimedia", logger, nil) avid := Plugs.NewAvid("avid", logger, nil) shure := Plugs.NewShure("shure", logger, nil) ps["yingxiaoqun"] = plug ps["radialeng"] = plugyo ps["maudio"] = maudio ps["jblpro"] = jblpro ps["ikmultimedia"] = ikmultimedia ps["avid"] = avid ps["shure"] = shure //启动调度器 scheduler := Scheduler.NewScheduler(9, logger, kvdb) out_chan, _ := scheduler.ConfigScheduler() scheduler.StartScheduler() //启动分发器 dispatcher := Dispatcher.NewDispatcher(out_chan, logger) dispatcher.ConfigureDispatcher(9, ps, scheduler) dispatcher.StartDispatcher() //读取种子url urls, err := ReadSeedUrls() if err != nil { fmt.Printf("[ERROR] ReadSeedUrls Error: %v\n", err) return } for k, v := range urls { for _, url := range v { scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k}) } } addr := fmt.Sprintf(":%d", 9999) go http.ListenAndServe(addr, router) //err = http.ListenAndServe(addr, router) //if err != nil { // logger.Error("Server start fail: %v", err) // os.Exit(1) //} fmt.Printf("START SERVER : %v\n", addr) interval, _ := configure.GetInterval() for { select { case <-time.After(time.Second * time.Duration(interval)): for k, v := range urls { for _, url := range v { scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k}) } } } } //out<-u.SpiderOut{Url:"http://www.yingxiaoqun.com111",Content:"",Header:""} u.SpiderSync.Wait() //time.Sleep(100*time.Second) }
func main() { fmt.Printf("init FalconEngine.....\n") //读取启动参数 var configFile string var search string var cores int var err error flag.StringVar(&configFile, "conf", "search.conf", "configure file full path") flag.StringVar(&search, "mode", "search", "start mode[ search | build ]") flag.IntVar(&cores, "core", 4, "cpu cores") flag.Parse() runtime.GOMAXPROCS(cores) //读取配置文件 configure, err := BaseFunctions.NewConfigure(configFile) if err != nil { fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err) return } //启动日志系统 logger, err := log4jzl.New("FalconEngine") if err != nil { fmt.Printf("[ERROR] Create logger Error: %v\n", err) //return } //初始化数据库适配器 dbAdaptor, err := BaseFunctions.NewDBAdaptor(configure, logger) if err != nil { fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err) return } defer dbAdaptor.Release() //初始化本地redis redisClient, err := BaseFunctions.NewRedisClient(configure, logger) if err != nil { fmt.Printf("[ERROR] Create redisClient Error: %v\n", err) return } defer redisClient.Release() if search == "search" { processor := &BaseFunctions.BaseProcessor{configure, logger, dbAdaptor, redisClient} bitmap := utils.NewBitmap() fields, err := configure.GetTableFields() if err != nil { logger.Error("%v", err) return } index_set := indexer.NewIndexSet(bitmap, logger) index_set.InitIndexSet(fields) searcher := NewSearcher(processor, index_set) // &Searcher{processor} data_chan := make(chan builder.UpdateInfo, 1000) updater := NewUpdater(processor, index_set, data_chan) updater.IncUpdating() router := &BaseFunctions.Router{configure, logger, map[string]BaseFunctions.FEProcessor{ "search": searcher, "update": updater, }} builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, index_set) builder.StartIncUpdate(data_chan) logger.Info("Server Start...") port, _ := configure.GetPort() addr := fmt.Sprintf(":%d", port) err = http.ListenAndServe(addr, router) if err != nil { logger.Error("Server start fail: %v", err) os.Exit(1) } } else if search == "build" { builder := NewBuilderEngine(configure, dbAdaptor, logger, redisClient, nil) builder.BuidingAllIndex() } else { logger.Error("Wrong start mode...only support [ search | build ]") } }