func router(w http.ResponseWriter, r *http.Request) {
// determine application db
appNum, err := shared.GetSwitch(switchDatabase)
if err != nil {
panic(fmt.Sprintf("Failed to query switch database for app db number in frontend.router: %v", err))
}
if appNum == 1 {
appNum = 2
} else {
appNum = 1
}
appDb, err = sql.Open("postgres", fmt.Sprintf(conn, appNum))
if err != nil {
panic(err)
}
defer appDb.Close()
// process request
path := strings.ToLower(r.URL.Path)
var matched bool
for _, tuple := range routing {
handler := tuple[1].(func(http.ResponseWriter, *http.Request, map[string]string))
if ro := route(tuple[0].(string)); ro.match(path) {
handler(w, r, ro.parse(path))
matched = true
}
}
if !matched {
fmt.Fprintf(w, "No route matched for:\n%q", r.URL.Path)
}
}
// flipSwitch changes the value stored in the 'switch db' from 1 to 2
// or from 2 to 1.
func flipSwitch(db *sql.DB) error {
currentSwitch, err := shared.GetSwitch(db)
if err != nil {
return err
}
var newSwitch int
if currentSwitch == 1 {
newSwitch = 2
} else {
newSwitch = 1
}
query := fmt.Sprintf(
"UPDATE switch_table SET switch_col = %d WHERE switch_col = %d",
newSwitch,
currentSwitch,
)
_, err = db.Exec(query)
if err != nil {
return err
}
return nil
}
func handleScrape(args []string) {
conf := parseConfig(args)
user := conf.DbLogin["user"]
password := conf.DbLogin["password"]
dbname := conf.DbLogin["dbname"]
for {
// scrape for each schedule specified in config
for _, schedule := range conf.Schedules {
// connect to switch db
switchDb, err := sql.Open("postgres", fmt.Sprintf(
"user=%s dbname=%s password=%s sslmode=require",
user,
fmt.Sprintf("goschedule_%s_switch", schedule["name"]),
password,
))
if err != nil {
fmt.Println(err)
os.Exit(1)
}
appNum, err := shared.GetSwitch(switchDb)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// reset app db
runSql("postgres", fmt.Sprintf(
"user=%s dbname=%s password=%s sslmode=require",
user,
dbname,
password),
fmt.Sprintf("DROP DATABASE goschedule_%s_app%d", schedule["name"], appNum),
fmt.Sprintf("CREATE DATABASE goschedule_%s_app%d", schedule["name"], appNum))
// connect to app db
appDb, err := sql.Open("postgres", fmt.Sprintf(
"user=%s dbname=%s password=%s sslmode=require",
conf.DbLogin["user"],
fmt.Sprintf("goschedule_%s_app%d", schedule["name"], appNum),
conf.DbLogin["password"],
))
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// load app db with schemas
runSql("postgres", fmt.Sprintf(
"user=%s dbname=%s password=%s sslmode=require",
user,
fmt.Sprintf("goschedule_%s_app%d", schedule["name"], appNum),
password,
), dbSetupStatements...)
// start scrape
start := time.Now()
fmt.Printf("Scraping %q using application database %d\n", schedule["url"], appNum)
backend.Scrape(schedule["url"], conf.DepartmentDescriptionIndex, appDb)
fmt.Println("Time taken:", time.Since(start))
// flip db switch
if err := flipSwitch(switchDb); err != nil {
fmt.Println(err)
os.Exit(1)
}
switchDb.Close()
fmt.Printf("Scrape for %q done\n", schedule["url"])
// close connection to app db
appDb.Close()
}
if !conf.LoopScraper {
break
}
time.Sleep(time.Duration(conf.ScraperTimeout) * time.Minute)
}
}