func main() {
var d *dispatcher.Dispatcher
/* Initialize the database */
//db, err := sql.Open("sqlite3", "scouting.db");
//if(err != nil) {
// log.Fatal(err);
//}
//defer db.Close();
fmt.Println("Version: " + g_version)
// Open the file
file, err := os.OpenFile("output.csv", os.O_RDWR|os.O_APPEND, 0660)
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
// Write the list of columns to the CSV file
/* columns := []string{"timestamp", "f_matchNumber", "f_teamNumber", "f_numAutoZone", "f_numStageBin", "f_numStepBin", "f_robotAuton", "f_toteAuton", "f_numOnStep", "f_numOnTop", "f_fouls", "f_stacks", "f_dead", "f_tipped", "f_tippedOtherRobot", "f_morePlayerStation", "f_notes"}
for i := 1; i < 5; i++ {
columns = append(columns, "f_" + strconv.Itoa(i) + "_litter");
columns = append(columns, "f_" + strconv.Itoa(i) + "_can");
columns = append(columns, "f_" + strconv.Itoa(i) + "_ntotes");
}
writer := csv.NewWriter(file);
writer.Write(columns);
writer.Flush(); */
// Create our list of columns from the first line of the CSV file
reader := csv.NewReader(file)
columns, err := reader.Read()
if err != nil {
fmt.Println(err)
return
}
// Seek to the end so we can begin writing
file.Seek(0, 2)
msgHandle := new(ReceiveMessageHandle)
msgHandle.columns = columns
msgHandle.file = file
//msgHandle.database = db;
/* Initialize the HTTP server */
d = dispatcher.NewDispatcher()
d.RegisterExpr("^/$", http.HandlerFunc(mainHandle))
d.RegisterExpr("^/form$", http.HandlerFunc(formHandle))
d.RegisterExpr("^/version.js$", http.HandlerFunc(versionHandle))
d.RegisterExpr("^/output.csv$", http.HandlerFunc(csvHandle))
d.RegisterExpr("^/write$", http.StripPrefix("/", msgHandle))
d.RegisterExpr("^/res", http.StripPrefix("/res/", http.FileServer(http.Dir("html/res/"))))
log.Fatal(http.ListenAndServe("127.0.0.1:8088", d))
}
func main() {
//var wg sync.WaitGroup
//读取启动参数
var configFile string
flag.StringVar(&configFile, "conf", "config.ini", "configure file full path")
flag.Parse()
fmt.Printf("Start AccipitrSpiderEngine...\n")
//启动日志系统
logger, err := log4jzl.New("AccipitrSpiderEngine")
if err != nil {
fmt.Printf("[ERROR] Create logger Error: %v\n", err)
//return
}
//读取配置文件
configure, err := u.NewConfigure(configFile)
if err != nil {
fmt.Printf("[ERROR] Parse Configure File Error: %v\n", err)
return
}
//初始化数据库适配器
/*
dbAdaptor, err := u.NewDBAdaptor(configure, logger)
if err != nil {
fmt.Printf("[ERROR] Create DB Adaptor Error: %v\n", err)
return
}
defer dbAdaptor.Release()
*/
//初始化KVDB数据库
DBname, _ := configure.GetKVDB()
var kvdb *u.PandionKV
if !Exist(fmt.Sprintf("./DB/%v.idx", DBname)) {
kvdb = u.NewPandionKV(DBname, logger)
} else {
kvdb = u.NewPandionKVWithFile(DBname, logger)
}
router := &Router{Kvdb: kvdb}
//初始化自定义的插件
ps := make(map[string]u.PlugInterface)
plug := Plugs.NewYXQ("yingxiaoqun", logger, nil) // dbAdaptor)
plugyo := Plugs.NewRadialeng("radialeng", logger, nil)
maudio := Plugs.NewMAudio("maudio", logger, nil)
jblpro := Plugs.NewJblpro("jblpor", logger, nil)
ikmultimedia := Plugs.NewIkmultimedia("ikmultimedia", logger, nil)
avid := Plugs.NewAvid("avid", logger, nil)
shure := Plugs.NewShure("shure", logger, nil)
ps["yingxiaoqun"] = plug
ps["radialeng"] = plugyo
ps["maudio"] = maudio
ps["jblpro"] = jblpro
ps["ikmultimedia"] = ikmultimedia
ps["avid"] = avid
ps["shure"] = shure
//启动调度器
scheduler := Scheduler.NewScheduler(9, logger, kvdb)
out_chan, _ := scheduler.ConfigScheduler()
scheduler.StartScheduler()
//启动分发器
dispatcher := Dispatcher.NewDispatcher(out_chan, logger)
dispatcher.ConfigureDispatcher(9, ps, scheduler)
dispatcher.StartDispatcher()
//读取种子url
urls, err := ReadSeedUrls()
if err != nil {
fmt.Printf("[ERROR] ReadSeedUrls Error: %v\n", err)
return
}
for k, v := range urls {
for _, url := range v {
scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k})
}
}
addr := fmt.Sprintf(":%d", 9999)
go http.ListenAndServe(addr, router)
//err = http.ListenAndServe(addr, router)
//if err != nil {
// logger.Error("Server start fail: %v", err)
// os.Exit(1)
//}
fmt.Printf("START SERVER : %v\n", addr)
interval, _ := configure.GetInterval()
for {
select {
case <-time.After(time.Second * time.Duration(interval)):
for k, v := range urls {
for _, url := range v {
scheduler.SendUrl(u.CrawlData{Url: url, Type: u.SEED_URL, HandlerName: k})
}
}
}
}
//out<-u.SpiderOut{Url:"http://www.yingxiaoqun.com111",Content:"",Header:""}
u.SpiderSync.Wait()
//time.Sleep(100*time.Second)
}