예제 #1
0
func setupAPITasks(app *server.App) {
	var AsyncAPI = app.AsyncAPI()

	profileSyncTask := AsyncAPI.Define("/tasks/profile/")
	profileSyncTask.Queue.Rate = "1/s"
	profileSyncTask.Queue.MaxConcurrentRequests = "1"
	profileSyncTask.OnTrigger(handlers.AdminGate)
	profileSyncTask.OnMonitor(handlers.AdminGate)
	profileSyncTask.OnProcess(server.AsyncTaskHandler(runTasksProfile))
	profileSyncTask.ScheduleCron(
		"Synchronize profiles",
		"every day 00:10",
	)

	crawlerAmebloEntryListsTask := AsyncAPI.Define("/tasks/crawlers/ameblo/entrylists/")
	crawlerAmebloEntryListsTask.Queue.Rate = "1/s"
	crawlerAmebloEntryListsTask.Queue.MaxConcurrentRequests = "1"
	crawlerAmebloEntryListsTask.OnTrigger(handlers.AdminGate)
	crawlerAmebloEntryListsTask.OnMonitor(handlers.AdminGate)
	crawlerAmebloEntryListsTask.OnProcess(server.AsyncTaskHandler(runTasksCrawlersAmebloEntryLists))
	crawlerAmebloEntryListsTask.ScheduleCron(
		"Crawl Ameblo Entry Lists",
		"every 30 minutes from 18:00 to 23:59",
		"every 30 minutes from 00:00 to 03:00",
		"every 3 hours from 03:00 to 17:59",
	)

	crawlerAmebloPostsTask := AsyncAPI.Define("/tasks/crawlers/ameblo/posts/")
	crawlerAmebloPostsTask.Queue.Rate = "1/s"
	crawlerAmebloPostsTask.Queue.MaxConcurrentRequests = "1"
	crawlerAmebloPostsTask.OnTrigger(handlers.AdminGate)
	crawlerAmebloPostsTask.OnMonitor(handlers.AdminGate)
	crawlerAmebloPostsTask.OnProcess(server.AsyncTaskHandler(runTasksCrawlersAmebloPosts))
	crawlerAmebloPostsTask.ScheduleCron(
		"Crawl Ameblo Posts",
		"every 30 minutes from 19:10 to 23:59",
		"every 30 minutes from 00:10 to 03:10",
		"every 2 hours from 04:10 to 19:09",
	)
}
예제 #2
0
func setupAPIIEPGExclusions(app *server.App) {
	var AsyncAPI = app.AsyncAPI()
	var API = app.API()

	API.GET("/iepg/exclusions/", middleware.EntityAll(IEPGExclusion.Query().Cache(_CacheAllIEPGExclusions)))

	exclusionTask := AsyncAPI.Define("/iepg/exclusions/task/")
	exclusionTask.Queue.Rate = "1/s"
	exclusionTask.Queue.MaxConcurrentRequests = "1"
	exclusionTask.OnTrigger(middleware.Gate("family"))
	exclusionTask.OnMonitor(middleware.Gate("family"))
	exclusionTask.OnProcess(server.AsyncTaskHandler(
		func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
			var list []*pt.IEPG
			records := recentRecordsQuery().MustExecute(req).Data.([]pt.IEPG)
			for i := range records {
				list = append(list, &records[i])
			}
			if err := applyExclusions(req, list); err != nil {
				return nil, err
			}
			IEPG.PutMulti().Cache(_CacheAllIEPGExclusions).MustUpdate(req, list)
			return nil, nil
		},
	))

	API.POST("/iepg/exclusions/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("cid").Required()
			v.Field("sid").Required()
			v.Field("weekday").Required()
			v.Field("title_text").Required()
			v.Field("title_text_type").Required()
		}),
		middleware.EntityPost(IEPGExclusion.Put().Cache(_CacheAllIEPGExclusions)),
	)

	API.DELETE("/iepg/exclusions/:id.json",
		middleware.Gate("family"),
		middleware.EntityDelete(IEPGExclusion.Delete().Cache(_CacheAllIEPGExclusions), "id"))
}
예제 #3
0
func setupAPIIEPGKeywords(app *server.App) {
	var API = app.API()

	API.GET("/iepg/keywords/",
		middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))

	API.POST("/iepg/keywords/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.Field("category").Required()
			v.IntField("scope").Required()
		}),
		middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))

	API.DELETE("/iepg/keywords/:id.json",
		middleware.Gate("family"),
		middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))

	API.POST("/iepg/keywords/preview/",
		middleware.Gate("family"),
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("keyword").Required()
			v.IntField("scope").Required()
		}),
		server.Handler(
			func(req *wcg.Request) response.Response {
				list, err := crawlKeyword(req, &pt.IEPGKeyword{
					Keyword: req.Form("keyword"),
					Scope:   pt.ParseIEPGCrawlerScope(req.Form("scope")),
				})
				if err != nil {
					return response.InternalServerError(req, err)
				}
				return response.NewJSONResponse(map[string]interface{}{
					"hits": list,
				})
			},
		))

	var AsyncAPI = app.AsyncAPI()

	keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
	keywordCrawlTask.Queue.Rate = "1/s"
	keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
	keywordCrawlTask.OnTrigger(middleware.Gate("family"))
	keywordCrawlTask.OnMonitor(middleware.Gate("family"))
	keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
		func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
			data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
			if data == nil {
				return nil, nil
			}
			var keywords = data.([]pt.IEPGKeyword)
			req.Logger.Infof("Crawling %d keywords", len(keywords))
			for _, key := range keywords {
				list, err := crawlKeyword(req, &key)
				if err != nil {
					return nil, err
				}
				IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
			}
			return nil, nil
		},
	))
	keywordCrawlTask.ScheduleCron(
		"Crawl IEPG keywords",
		"every 1 hours",
	)
}