func recentRecordsQuery() *entities.Query { return IEPG.Query().Filter("EndAt >=", request.Value(func(*wcg.Request) interface{} { return lib.Today().Add(-recentRecordsTimeWindow) })).Filter("EndAt <", request.Value(func(*wcg.Request) interface{} { return lib.Today().Add(recentRecordsTimeWindow * 2) })).Order("-EndAt") }
func setupAPITasks(app *server.App) { var API = app.API() API.GET("/tasks/", server.Handler( middleware.EntityQuery( entities.AsyncAPITask.Query().Filter( "UpdatedAt >=", request.Value(func(req *wcg.Request) interface{} { return wcg.ParseDateOr(req.Query("since"), lib.Now().Add(-_TaskExpirationDays)) }), ).Filter( "UpdatedAt <=", request.Value(func(req *wcg.Request) interface{} { return wcg.ParseDateOr(req.Query("until"), lib.Now()) }), ), ), )) }
func setupAPICrawlerSettings(app *server.App) { var API = app.API() var urlValidator = server.Handler(func(req *wcg.Request) response.Response { _, err := url.Parse(req.Param("url")) if err != nil { return response.APINotFound } return nil }) API.GET("/crawlersettings/:type/", middleware.EntityAll(CrawlerSettings.Query().Filter("Type=", request.Value(func(req *wcg.Request) interface{} { t, _ := hplink.ParseCrawlerSettingsType(req.Param("type")) if t == hplink.CrawlerSettingsTypeUnknown { return entities.FilterValueSkip } return t }))), ) API.GET("/crawlersettings/ameblo/stats/", server.Handler(func(req *wcg.Request) response.Response { type post struct { URL string `json:"url"` At time.Time `json:"at"` } type stats struct { URL string `json:"url"` FirstPost *post `json:"first_post,omitempty"` LastPost *post `json:"last_post,omitempty"` TotalPosts int `json:"total_posts"` CrawledPosts int `json:"crawled_posts"` ImageFailurePosts int `json:"image_failure_posts"` } p := CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req) settings := p.Data.([]hplink.CrawlerSettings) s := make([]stats, len(settings)) if err := iterator.ParallelSlice(settings, func(i int, v *hplink.CrawlerSettings) error { s[i].URL = v.URL s[i].TotalPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).MustCount(req) if s[i].TotalPosts > 0 { s[i].CrawledPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("IsContentsCrawled=", true).MustCount(req) s[i].ImageFailurePosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("Images.Height=", 0).MustCount(req) pf := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("PostAt").Limit(1).MustExecute(req) pl := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("-PostAt").Limit(1).MustExecute(req) first := pf.Head().(*hplink.AmebloPost) last := pl.Head().(*hplink.AmebloPost) s[i].FirstPost = &post{ URL: first.URL, At: first.PostAt, } s[i].LastPost = &post{ URL: last.URL, At: last.PostAt, } return nil } return nil }); err != nil { panic(err) } return response.NewJSONResponse(s) }), ) API.GET("/crawlersettings/:url.json", middleware.EntityGet(CrawlerSettings.Get(), "url"), ) API.PUT("/crawlersettings/:url.json", urlValidator, middleware.ParseForm(func(v *validators.FormValidator) { v.Field("artist_key").Required() }), middleware.EntityPutOrCreate( CrawlerSettings.Put(), "url", ), ) API.DELETE("/crawlersettings/:url.json", urlValidator, middleware.EntityDelete(CrawlerSettings.Delete(), "url"), ) }