Example #1
0
func main() {

	var err error
	logger = getLogger()
	appConf, err = config.ParseConfig("../conf/app_conf.ini")
	if err != nil {
		fmt.Println(err)
		return
	}

	port := flag.String("p", "9090", "port to run the web server")
	toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database")

	flag.Parse()

	if *toolMode {
		// 数据转换存储方式
		err := tools.StorageTransform(appConf)
		tools.CheckError(err)
	} else {
		// 存储连接初始化
		db, err = applicationDB.InitStoreConnection(appConf)
		if err != nil {
			logger.Fatalln(err)
			os.Exit(1)
		}
		defer db.Close()

		// 请求路由
		http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("../static/"))))
		http.HandleFunc("/applyvport", applyVPort)
		http.HandleFunc("/edittask", editTask)
		http.HandleFunc("/listenlist", getListenList)
		http.HandleFunc("/dellistentask", delListenTask)
		http.HandleFunc("/applyconf", applyConf)
		http.HandleFunc("/statspage", statsPage)
		http.HandleFunc("/", getHomePage)

		// 启动http服务
		err = http.ListenAndServe(":"+*port, nil)
		if err != nil {
			logger.Fatalln("ListenAndServe: ", err)
		}
	}
}
Example #2
0
func autocrawl(needAll bool, crawlConfFile string, whichSite string) {

	_, err := config.ParseConfig(crawlConfFile, &websites)
	if err != nil {
		log.Fatalln("parse crawl config error:", err)
	}

	if needAll {
		// 全量
		for website, wbconf := range websites {
			if whichSite != "" && whichSite != website {
				continue
			}

			logger.Infoln("all crawl", website)
			go doCrawl(wbconf, true)
		}
	}

	// 定时增量
	c := cron.New()
	c.AddFunc(config.Config["crawl_spec"], func() {
		// 抓取 reddit
		go service.ParseReddit("")

		// 抓取 www.oschina.net/project
		go service.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time")

		for website, wbconf := range websites {
			if whichSite != "" && whichSite != website {
				continue
			}

			logger.Infoln("do crawl", website)
			go doCrawl(wbconf, false)
		}
	})
	c.Start()
}
Example #3
0
func main() {

	programName := os.Args[0]
	absPath, _ := filepath.Abs(programName)
	haproxyConsoleRoot = path.Dir(path.Dir(absPath))

	port := flag.String("p", "9090", "port to run the web server")
	configuration := flag.String("config", "", "path to the app config file")
	toolMode := flag.Bool("t", false, "run this program as a tool to export data from database to json or from json to database")

	flag.Parse()

	configPath := *configuration

	if configPath == "" {
		defaultConfigPath := haproxyConsoleRoot + "/conf/app_conf.ini"
		fmt.Printf("You not set the configPath, so try to use the default config path: %s\n", defaultConfigPath)
		if _, e := os.Stat(defaultConfigPath); os.IsNotExist(e) {
			fmt.Println("Not Exit default config file")
			os.Exit(1)
		} else {
			configPath = defaultConfigPath
		}
	}

	var err error
	logger = getLogger()

	appConf, err = config.ParseConfig(configPath)
	if err != nil {
		fmt.Println(err)
		return
	}

	if *toolMode {
		// 数据转换存储方式
		err := tools.StorageTransform(appConf)
		tools.CheckError(err)
	} else {
		// 存储连接初始化
		db, err = applicationDB.InitStoreConnection(appConf)
		if err != nil {
			logger.Fatalln(err)
			os.Exit(1)
		}
		defer db.Close()

		// 请求路由
		http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(haproxyConsoleRoot+"/static/"))))
		// basic auth user: admin, passwd: 1qazXSW@
		authenticator := auth.NewBasicAuthenticator("HAProxyConsole", auth.HtpasswdFileProvider(haproxyConsoleRoot+"/conf/md5passwd"))
		http.HandleFunc("/applyvport", authenticator.Wrap(applyVPort))
		http.HandleFunc("/edittask", authenticator.Wrap(editTask))
		http.HandleFunc("/listenlist", authenticator.Wrap(getListenList))
		http.HandleFunc("/dellistentask", authenticator.Wrap(delListenTask))
		http.HandleFunc("/applyconf", authenticator.Wrap(applyConf))
		http.HandleFunc("/statspage", statsPage)
		http.HandleFunc("/", authenticator.Wrap(getHomePage))

		// 启动http服务
		err = http.ListenAndServe(":"+*port, nil)
		if err != nil {
			logger.Fatalln("ListenAndServe: ", err)
		}
	}
}