func main() {
var err error
logger = getLogger()
appConf, err = config.ParseConfig("../conf/app_conf.ini")
if err != nil {
fmt.Println(err)
return
}
port := flag.String("p", "9090", "port to run the web server")
toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database")
flag.Parse()
if *toolMode {
// 数据转换存储方式
err := tools.StorageTransform(appConf)
tools.CheckError(err)
} else {
// 存储连接初始化
db, err = applicationDB.InitStoreConnection(appConf)
if err != nil {
logger.Fatalln(err)
os.Exit(1)
}
defer db.Close()
// 请求路由
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("../static/"))))
http.HandleFunc("/applyvport", applyVPort)
http.HandleFunc("/edittask", editTask)
http.HandleFunc("/listenlist", getListenList)
http.HandleFunc("/dellistentask", delListenTask)
http.HandleFunc("/applyconf", applyConf)
http.HandleFunc("/statspage", statsPage)
http.HandleFunc("/", getHomePage)
// 启动http服务
err = http.ListenAndServe(":"+*port, nil)
if err != nil {
logger.Fatalln("ListenAndServe: ", err)
}
}
}
func autocrawl(needAll bool, crawlConfFile string, whichSite string) {
_, err := config.ParseConfig(crawlConfFile, &websites)
if err != nil {
log.Fatalln("parse crawl config error:", err)
}
if needAll {
// 全量
for website, wbconf := range websites {
if whichSite != "" && whichSite != website {
continue
}
logger.Infoln("all crawl", website)
go doCrawl(wbconf, true)
}
}
// 定时增量
c := cron.New()
c.AddFunc(config.Config["crawl_spec"], func() {
// 抓取 reddit
go service.ParseReddit("")
// 抓取 www.oschina.net/project
go service.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time")
for website, wbconf := range websites {
if whichSite != "" && whichSite != website {
continue
}
logger.Infoln("do crawl", website)
go doCrawl(wbconf, false)
}
})
c.Start()
}
func main() {
programName := os.Args[0]
absPath, _ := filepath.Abs(programName)
haproxyConsoleRoot = path.Dir(path.Dir(absPath))
port := flag.String("p", "9090", "port to run the web server")
configuration := flag.String("config", "", "path to the app config file")
toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database")
flag.Parse()
configPath := *configuration
if configPath == "" {
defaultConfigPath := haproxyConsoleRoot + "/conf/app_conf.ini"
fmt.Printf("You not set the configPath, so try to use the default config path: %s\n", defaultConfigPath)
if _, e := os.Stat(defaultConfigPath); os.IsNotExist(e) {
fmt.Println("Not Exit default config file")
os.Exit(1)
} else {
configPath = defaultConfigPath
}
}
var err error
logger = getLogger()
appConf, err = config.ParseConfig(configPath)
if err != nil {
fmt.Println(err)
return
}
if *toolMode {
// 数据转换存储方式
err := tools.StorageTransform(appConf)
tools.CheckError(err)
} else {
// 存储连接初始化
db, err = applicationDB.InitStoreConnection(appConf)
if err != nil {
logger.Fatalln(err)
os.Exit(1)
}
defer db.Close()
// 请求路由
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(haproxyConsoleRoot+"/static/"))))
// basic auth user: admin, passwd: 1qazXSW@
authenticator := auth.NewBasicAuthenticator("HAProxyConsole", auth.HtpasswdFileProvider(haproxyConsoleRoot+"/conf/md5passwd"))
http.HandleFunc("/applyvport", authenticator.Wrap(applyVPort))
http.HandleFunc("/edittask", authenticator.Wrap(editTask))
http.HandleFunc("/listenlist", authenticator.Wrap(getListenList))
http.HandleFunc("/dellistentask", authenticator.Wrap(delListenTask))
http.HandleFunc("/applyconf", authenticator.Wrap(applyConf))
http.HandleFunc("/statspage", statsPage)
http.HandleFunc("/", authenticator.Wrap(getHomePage))
// 启动http服务
err = http.ListenAndServe(":"+*port, nil)
if err != nil {
logger.Fatalln("ListenAndServe: ", err)
}
}
}