Example #1
0
func main() {
	var d *dispatcher.Dispatcher

	/* Initialize the database */
	//db, err := sql.Open("sqlite3", "scouting.db");
	//if(err != nil) {
	//  log.Fatal(err);
	//}
	//defer db.Close();

	fmt.Println("Version: " + g_version)

	// Open the file
	file, err := os.OpenFile("output.csv", os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer file.Close()

	// Write the list of columns to the CSV file
	/* columns := []string{"timestamp", "f_matchNumber", "f_teamNumber", "f_numAutoZone", "f_numStageBin", "f_numStepBin", "f_robotAuton", "f_toteAuton", "f_numOnStep", "f_numOnTop", "f_fouls", "f_stacks", "f_dead", "f_tipped", "f_tippedOtherRobot", "f_morePlayerStation", "f_notes"}

	for i := 1; i < 5; i++ {
	  columns = append(columns, "f_" + strconv.Itoa(i) + "_litter");
	  columns = append(columns, "f_" + strconv.Itoa(i) + "_can");
	  columns = append(columns, "f_" + strconv.Itoa(i) + "_ntotes");
	}
	writer := csv.NewWriter(file);
	writer.Write(columns);
	writer.Flush(); */

	// Create our list of columns from the first line of the CSV file
	reader := csv.NewReader(file)
	columns, err := reader.Read()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Seek to the end so we can begin writing
	file.Seek(0, 2)

	msgHandle := new(ReceiveMessageHandle)
	msgHandle.columns = columns
	msgHandle.file = file
	//msgHandle.database = db;

	/* Initialize the HTTP server */
	d = dispatcher.NewDispatcher()
	d.RegisterExpr("^/$", http.HandlerFunc(mainHandle))
	d.RegisterExpr("^/form$", http.HandlerFunc(formHandle))
	d.RegisterExpr("^/version.js$", http.HandlerFunc(versionHandle))
	d.RegisterExpr("^/output.csv$", http.HandlerFunc(csvHandle))
	d.RegisterExpr("^/write$", http.StripPrefix("/", msgHandle))
	d.RegisterExpr("^/res", http.StripPrefix("/res/", http.FileServer(http.Dir("html/res/"))))

	log.Fatal(http.ListenAndServe("127.0.0.1:8088", d))
}
func main() {

	//var wg sync.WaitGroup

	//读取启动参数
	var configFile string
	flag.StringVar(&configFile, "conf", "config.ini", "configure file full path")
	flag.Parse()

	fmt.Printf("Start AccipitrSpiderEngine...\n")

	//启动日志系统
	logger, err := log4jzl.New("AccipitrSpiderEngine")
	if err != nil {
		fmt.Printf("[ERROR] Create logger Error: %v\n", err)
		//return
	}

	//读取配置文件
	configure, err := u.NewConfigure(configFile)
	if err != nil {
		fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err)
		return
	}

	//初始化数据库适配器
	/*
		dbAdaptor, err := u.NewDBAdaptor(configure, logger)
		if err != nil {
			fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err)
			return
		}
		defer dbAdaptor.Release()
	*/
	//初始化KVDB数据库
	DBname, _ := configure.GetKVDB()
	var kvdb *u.PandionKV
	if !Exist(fmt.Sprintf("./DB/%v.idx", DBname)) {
		kvdb = u.NewPandionKV(DBname, logger)
	} else {
		kvdb = u.NewPandionKVWithFile(DBname, logger)
	}

	router := &Router{Kvdb: kvdb}

	//初始化自定义的插件
	ps := make(map[string]u.PlugInterface)
	plug := Plugs.NewYXQ("yingxiaoqun", logger, nil) // dbAdaptor)
	plugyo := Plugs.NewRadialeng("radialeng", logger, nil)
	maudio := Plugs.NewMAudio("maudio", logger, nil)
	jblpro := Plugs.NewJblpro("jblpor", logger, nil)
	ikmultimedia := Plugs.NewIkmultimedia("ikmultimedia", logger, nil)
	avid := Plugs.NewAvid("avid", logger, nil)
	shure := Plugs.NewShure("shure", logger, nil)
	ps["yingxiaoqun"] = plug
	ps["radialeng"] = plugyo
	ps["maudio"] = maudio
	ps["jblpro"] = jblpro
	ps["ikmultimedia"] = ikmultimedia
	ps["avid"] = avid
	ps["shure"] = shure
	//启动调度器
	scheduler := Scheduler.NewScheduler(9, logger, kvdb)
	out_chan, _ := scheduler.ConfigScheduler()
	scheduler.StartScheduler()

	//启动分发器
	dispatcher := Dispatcher.NewDispatcher(out_chan, logger)
	dispatcher.ConfigureDispatcher(9, ps, scheduler)
	dispatcher.StartDispatcher()

	//读取种子url
	urls, err := ReadSeedUrls()
	if err != nil {
		fmt.Printf("[ERROR] ReadSeedUrls Error: %v\n", err)
		return
	}

	for k, v := range urls {
		for _, url := range v {
			scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k})
		}
	}

	addr := fmt.Sprintf(":%d", 9999)
	go http.ListenAndServe(addr, router)
	//err = http.ListenAndServe(addr, router)
	//if err != nil {
	//	logger.Error("Server start fail: %v", err)
	//	os.Exit(1)
	//}
	fmt.Printf("START SERVER : %v\n", addr)

	interval, _ := configure.GetInterval()
	for {
		select {
		case <-time.After(time.Second * time.Duration(interval)):
			for k, v := range urls {
				for _, url := range v {
					scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k})
				}
			}

		}
	}

	//out<-u.SpiderOut{Url:"http://www.yingxiaoqun.com111",Content:"",Header:""}
	u.SpiderSync.Wait()
	//time.Sleep(100*time.Second)

}