func main() { var err error logger = getLogger() appConf, err = config.ParseConfig("../conf/app_conf.ini") if err != nil { fmt.Println(err) return } port := flag.String("p", "9090", "port to run the web server") toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database") flag.Parse() if *toolMode { // 数据转换存储方式 err := tools.StorageTransform(appConf) tools.CheckError(err) } else { // 存储连接初始化 db, err = applicationDB.InitStoreConnection(appConf) if err != nil { logger.Fatalln(err) os.Exit(1) } defer db.Close() // 请求路由 http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("../static/")))) http.HandleFunc("/applyvport", applyVPort) http.HandleFunc("/edittask", editTask) http.HandleFunc("/listenlist", getListenList) http.HandleFunc("/dellistentask", delListenTask) http.HandleFunc("/applyconf", applyConf) http.HandleFunc("/statspage", statsPage) http.HandleFunc("/", getHomePage) // 启动http服务 err = http.ListenAndServe(":"+*port, nil) if err != nil { logger.Fatalln("ListenAndServe: ", err) } } }
func autocrawl(needAll bool, crawlConfFile string, whichSite string) { _, err := config.ParseConfig(crawlConfFile, &websites) if err != nil { log.Fatalln("parse crawl config error:", err) } if needAll { // 全量 for website, wbconf := range websites { if whichSite != "" && whichSite != website { continue } logger.Infoln("all crawl", website) go doCrawl(wbconf, true) } } // 定时增量 c := cron.New() c.AddFunc(config.Config["crawl_spec"], func() { // 抓取 reddit go service.ParseReddit("") // 抓取 www.oschina.net/project go service.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time") for website, wbconf := range websites { if whichSite != "" && whichSite != website { continue } logger.Infoln("do crawl", website) go doCrawl(wbconf, false) } }) c.Start() }
func main() { programName := os.Args[0] absPath, _ := filepath.Abs(programName) haproxyConsoleRoot = path.Dir(path.Dir(absPath)) port := flag.String("p", "9090", "port to run the web server") configuration := flag.String("config", "", "path to the app config file") toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database") flag.Parse() configPath := *configuration if configPath == "" { defaultConfigPath := haproxyConsoleRoot + "/conf/app_conf.ini" fmt.Printf("You not set the configPath, so try to use the default config path: %s\n", defaultConfigPath) if _, e := os.Stat(defaultConfigPath); os.IsNotExist(e) { fmt.Println("Not Exit default config file") os.Exit(1) } else { configPath = defaultConfigPath } } var err error logger = getLogger() appConf, err = config.ParseConfig(configPath) if err != nil { fmt.Println(err) return } if *toolMode { // 数据转换存储方式 err := tools.StorageTransform(appConf) tools.CheckError(err) } else { // 存储连接初始化 db, err = applicationDB.InitStoreConnection(appConf) if err != nil { logger.Fatalln(err) os.Exit(1) } defer db.Close() // 请求路由 http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(haproxyConsoleRoot+"/static/")))) // basic auth user: admin, passwd: 1qazXSW@ authenticator := auth.NewBasicAuthenticator("HAProxyConsole", auth.HtpasswdFileProvider(haproxyConsoleRoot+"/conf/md5passwd")) http.HandleFunc("/applyvport", authenticator.Wrap(applyVPort)) http.HandleFunc("/edittask", authenticator.Wrap(editTask)) http.HandleFunc("/listenlist", authenticator.Wrap(getListenList)) http.HandleFunc("/dellistentask", authenticator.Wrap(delListenTask)) http.HandleFunc("/applyconf", authenticator.Wrap(applyConf)) http.HandleFunc("/statspage", statsPage) http.HandleFunc("/", authenticator.Wrap(getHomePage)) // 启动http服务 err = http.ListenAndServe(":"+*port, nil) if err != nil { logger.Fatalln("ListenAndServe: ", err) } } }