示例#1
0
func init() {
	entities.RegisterValueParser(pt.IEPGCrawlerScope(0), func(s string) (interface{}, error) {
		return pt.ParseIEPGCrawlerScope(s), nil
	})
	entities.RegisterValueParser(pt.IEPGExclusionTextType(0), func(s string) (interface{}, error) {
		return pt.ParseIEPGExclusionTextType(s), nil
	})

	Channel = entities.RegisterKind("Channel", _AppNamespace, pt.Channel{})
	Channel.BeforeSave = func(ent interface{}, req *wcg.Request) error {
		v := ent.(*pt.Channel)
		v.ID = fmt.Sprintf("%s.%s", v.CID, v.SID)
		return nil
	}

	IEPG = entities.RegisterKind("IEPG", _AppNamespace, pt.IEPG{})

	IEPGKeyword = entities.RegisterKind("IEPGKeyword", _AppNamespace, pt.IEPGKeyword{})
	IEPGKeyword.BeforeSave = func(ent interface{}, req *wcg.Request) error {
		v := ent.(*pt.IEPGKeyword)
		v.Keyword = strings.TrimSpace(v.Keyword)
		return nil
	}

	IEPGExclusion = entities.RegisterKind("IEPGExclusion", _AppNamespace, pt.IEPGExclusion{})
	MessengerNotification = entities.RegisterKind("MessengerNotification", _AppNamespace, pt.MessengerNotification{})
	EmailNotification = entities.RegisterKind("EmailNotification", _AppNamespace, pt.EmailNotification{})
}
示例#2
0
func setupAPIIEPGKeywords(app *server.App) {
	var API = app.API()

	API.GET("/iepg/keywords/",
		middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))

	API.POST("/iepg/keywords/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.Field("category").Required()
			v.IntField("scope").Required()
		}),
		middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))

	API.DELETE("/iepg/keywords/:id.json",
		middleware.Gate("family"),
		middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))

	API.POST("/iepg/keywords/preview/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.IntField("scope").Required()
		}),
		server.Handler(
			func(req *wcg.Request) response.Response {
				list, err := crawlKeyword(req, &pt.IEPGKeyword{
					Keyword: req.Form("keyword"),
					Scope:   pt.ParseIEPGCrawlerScope(req.Form("scope")),
				})
				if err != nil {
					return response.InternalServerError(req, err)
				}
				return response.NewJSONResponse(map[string]interface{}{
					"hits": list,
				})
			},
		))

	var AsyncAPI = app.AsyncAPI()

	keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
	keywordCrawlTask.Queue.Rate = "1/s"
	keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
	keywordCrawlTask.OnTrigger(middleware.Gate("family"))
	keywordCrawlTask.OnMonitor(middleware.Gate("family"))
	keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
		func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
			data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
			if data == nil {
				return nil, nil
			}
			var keywords = data.([]pt.IEPGKeyword)
			req.Logger.Infof("Crawling %d keywords", len(keywords))
			for _, key := range keywords {
				list, err := crawlKeyword(req, &key)
				if err != nil {
					return nil, err
				}
				IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
			}
			return nil, nil
		},
	))
	keywordCrawlTask.ScheduleCron(
		"Crawl IEPG keywords",
		"every 1 hours",
	)
}