func init() {
entities.RegisterValueParser(pt.IEPGCrawlerScope(0), func(s string) (interface{}, error) {
return pt.ParseIEPGCrawlerScope(s), nil
})
entities.RegisterValueParser(pt.IEPGExclusionTextType(0), func(s string) (interface{}, error) {
return pt.ParseIEPGExclusionTextType(s), nil
})
Channel = entities.RegisterKind("Channel", _AppNamespace, pt.Channel{})
Channel.BeforeSave = func(ent interface{}, req *wcg.Request) error {
v := ent.(*pt.Channel)
v.ID = fmt.Sprintf("%s.%s", v.CID, v.SID)
return nil
}
IEPG = entities.RegisterKind("IEPG", _AppNamespace, pt.IEPG{})
IEPGKeyword = entities.RegisterKind("IEPGKeyword", _AppNamespace, pt.IEPGKeyword{})
IEPGKeyword.BeforeSave = func(ent interface{}, req *wcg.Request) error {
v := ent.(*pt.IEPGKeyword)
v.Keyword = strings.TrimSpace(v.Keyword)
return nil
}
IEPGExclusion = entities.RegisterKind("IEPGExclusion", _AppNamespace, pt.IEPGExclusion{})
MessengerNotification = entities.RegisterKind("MessengerNotification", _AppNamespace, pt.MessengerNotification{})
EmailNotification = entities.RegisterKind("EmailNotification", _AppNamespace, pt.EmailNotification{})
}
func setupAPIIEPGKeywords(app *server.App) {
var API = app.API()
API.GET("/iepg/keywords/",
middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))
API.POST("/iepg/keywords/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.Field("category").Required()
v.IntField("scope").Required()
}),
middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))
API.DELETE("/iepg/keywords/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))
API.POST("/iepg/keywords/preview/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.IntField("scope").Required()
}),
server.Handler(
func(req *wcg.Request) response.Response {
list, err := crawlKeyword(req, &pt.IEPGKeyword{
Keyword: req.Form("keyword"),
Scope: pt.ParseIEPGCrawlerScope(req.Form("scope")),
})
if err != nil {
return response.InternalServerError(req, err)
}
return response.NewJSONResponse(map[string]interface{}{
"hits": list,
})
},
))
var AsyncAPI = app.AsyncAPI()
keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
keywordCrawlTask.Queue.Rate = "1/s"
keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
keywordCrawlTask.OnTrigger(middleware.Gate("family"))
keywordCrawlTask.OnMonitor(middleware.Gate("family"))
keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
if data == nil {
return nil, nil
}
var keywords = data.([]pt.IEPGKeyword)
req.Logger.Infof("Crawling %d keywords", len(keywords))
for _, key := range keywords {
list, err := crawlKeyword(req, &key)
if err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
}
return nil, nil
},
))
keywordCrawlTask.ScheduleCron(
"Crawl IEPG keywords",
"every 1 hours",
)
}
