func setupAPITasks(app *server.App) {
var AsyncAPI = app.AsyncAPI()
profileSyncTask := AsyncAPI.Define("/tasks/profile/")
profileSyncTask.Queue.Rate = "1/s"
profileSyncTask.Queue.MaxConcurrentRequests = "1"
profileSyncTask.OnTrigger(handlers.AdminGate)
profileSyncTask.OnMonitor(handlers.AdminGate)
profileSyncTask.OnProcess(server.AsyncTaskHandler(runTasksProfile))
profileSyncTask.ScheduleCron(
"Synchronize profiles",
"every day 00:10",
)
crawlerAmebloEntryListsTask := AsyncAPI.Define("/tasks/crawlers/ameblo/entrylists/")
crawlerAmebloEntryListsTask.Queue.Rate = "1/s"
crawlerAmebloEntryListsTask.Queue.MaxConcurrentRequests = "1"
crawlerAmebloEntryListsTask.OnTrigger(handlers.AdminGate)
crawlerAmebloEntryListsTask.OnMonitor(handlers.AdminGate)
crawlerAmebloEntryListsTask.OnProcess(server.AsyncTaskHandler(runTasksCrawlersAmebloEntryLists))
crawlerAmebloEntryListsTask.ScheduleCron(
"Crawl Ameblo Entry Lists",
"every 30 minutes from 18:00 to 23:59",
"every 30 minutes from 00:00 to 03:00",
"every 3 hours from 03:00 to 17:59",
)
crawlerAmebloPostsTask := AsyncAPI.Define("/tasks/crawlers/ameblo/posts/")
crawlerAmebloPostsTask.Queue.Rate = "1/s"
crawlerAmebloPostsTask.Queue.MaxConcurrentRequests = "1"
crawlerAmebloPostsTask.OnTrigger(handlers.AdminGate)
crawlerAmebloPostsTask.OnMonitor(handlers.AdminGate)
crawlerAmebloPostsTask.OnProcess(server.AsyncTaskHandler(runTasksCrawlersAmebloPosts))
crawlerAmebloPostsTask.ScheduleCron(
"Crawl Ameblo Posts",
"every 30 minutes from 19:10 to 23:59",
"every 30 minutes from 00:10 to 03:10",
"every 2 hours from 04:10 to 19:09",
)
}
func setupAPIIEPGExclusions(app *server.App) {
var AsyncAPI = app.AsyncAPI()
var API = app.API()
API.GET("/iepg/exclusions/", middleware.EntityAll(IEPGExclusion.Query().Cache(_CacheAllIEPGExclusions)))
exclusionTask := AsyncAPI.Define("/iepg/exclusions/task/")
exclusionTask.Queue.Rate = "1/s"
exclusionTask.Queue.MaxConcurrentRequests = "1"
exclusionTask.OnTrigger(middleware.Gate("family"))
exclusionTask.OnMonitor(middleware.Gate("family"))
exclusionTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
var list []*pt.IEPG
records := recentRecordsQuery().MustExecute(req).Data.([]pt.IEPG)
for i := range records {
list = append(list, &records[i])
}
if err := applyExclusions(req, list); err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGExclusions).MustUpdate(req, list)
return nil, nil
},
))
API.POST("/iepg/exclusions/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("cid").Required()
v.Field("sid").Required()
v.Field("weekday").Required()
v.Field("title_text").Required()
v.Field("title_text_type").Required()
}),
middleware.EntityPost(IEPGExclusion.Put().Cache(_CacheAllIEPGExclusions)),
)
API.DELETE("/iepg/exclusions/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGExclusion.Delete().Cache(_CacheAllIEPGExclusions), "id"))
}
func setupAPIIEPGKeywords(app *server.App) {
var API = app.API()
API.GET("/iepg/keywords/",
middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))
API.POST("/iepg/keywords/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.Field("category").Required()
v.IntField("scope").Required()
}),
middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))
API.DELETE("/iepg/keywords/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))
API.POST("/iepg/keywords/preview/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.IntField("scope").Required()
}),
server.Handler(
func(req *wcg.Request) response.Response {
list, err := crawlKeyword(req, &pt.IEPGKeyword{
Keyword: req.Form("keyword"),
Scope: pt.ParseIEPGCrawlerScope(req.Form("scope")),
})
if err != nil {
return response.InternalServerError(req, err)
}
return response.NewJSONResponse(map[string]interface{}{
"hits": list,
})
},
))
var AsyncAPI = app.AsyncAPI()
keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
keywordCrawlTask.Queue.Rate = "1/s"
keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
keywordCrawlTask.OnTrigger(middleware.Gate("family"))
keywordCrawlTask.OnMonitor(middleware.Gate("family"))
keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
if data == nil {
return nil, nil
}
var keywords = data.([]pt.IEPGKeyword)
req.Logger.Infof("Crawling %d keywords", len(keywords))
for _, key := range keywords {
list, err := crawlKeyword(req, &key)
if err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
}
return nil, nil
},
))
keywordCrawlTask.ScheduleCron(
"Crawl IEPG keywords",
"every 1 hours",
)
}
