func setupAPIAPITokens(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// APIToken
API.GET("/api_tokens/",
middleware.EntityAll(entities.APIToken.Query()))
API.GET("/api_tokens/:token.json",
middleware.EntityGet(entities.APIToken.Get().Cache(true), "token"))
API.POST("/api_tokens/",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("description").Required()
}),
middleware.EntityPost(entities.APIToken.Put()))
API.PUT("/api_tokens/:token.json",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("description").Required()
}),
middleware.EntityPut(entities.APIToken.Put(), "token"))
API.DELETE("/api_tokens/:token.json",
middleware.EntityDelete(entities.APIToken.Delete(), "token"))
}
func setupAPIIEPGExclusions(app *server.App) {
var AsyncAPI = app.AsyncAPI()
var API = app.API()
API.GET("/iepg/exclusions/", middleware.EntityAll(IEPGExclusion.Query().Cache(_CacheAllIEPGExclusions)))
exclusionTask := AsyncAPI.Define("/iepg/exclusions/task/")
exclusionTask.Queue.Rate = "1/s"
exclusionTask.Queue.MaxConcurrentRequests = "1"
exclusionTask.OnTrigger(middleware.Gate("family"))
exclusionTask.OnMonitor(middleware.Gate("family"))
exclusionTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
var list []*pt.IEPG
records := recentRecordsQuery().MustExecute(req).Data.([]pt.IEPG)
for i := range records {
list = append(list, &records[i])
}
if err := applyExclusions(req, list); err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGExclusions).MustUpdate(req, list)
return nil, nil
},
))
API.POST("/iepg/exclusions/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("cid").Required()
v.Field("sid").Required()
v.Field("weekday").Required()
v.Field("title_text").Required()
v.Field("title_text_type").Required()
}),
middleware.EntityPost(IEPGExclusion.Put().Cache(_CacheAllIEPGExclusions)),
)
API.DELETE("/iepg/exclusions/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGExclusion.Delete().Cache(_CacheAllIEPGExclusions), "id"))
}
func setupAPIChannels(app *server.App) {
var API = app.API()
API.GET("/channels/",
middleware.EntityAll(Channel.Query().Cache(_CacheAllChannels)))
API.POST("/channels/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("cid").Required()
v.Field("sid").Required()
v.Field("name").Required()
v.Field("iepg_station_id").Required()
}),
middleware.EntityPost(Channel.Put().Cache(_CacheAllChannels)))
API.DELETE("/channels/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(Channel.Delete().Cache(_CacheAllChannels), "id"))
}
func setupAPIIEPGKeywords(app *server.App) {
var API = app.API()
API.GET("/iepg/keywords/",
middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))
API.POST("/iepg/keywords/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.Field("category").Required()
v.IntField("scope").Required()
}),
middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))
API.DELETE("/iepg/keywords/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))
API.POST("/iepg/keywords/preview/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.IntField("scope").Required()
}),
server.Handler(
func(req *wcg.Request) response.Response {
list, err := crawlKeyword(req, &pt.IEPGKeyword{
Keyword: req.Form("keyword"),
Scope: pt.ParseIEPGCrawlerScope(req.Form("scope")),
})
if err != nil {
return response.InternalServerError(req, err)
}
return response.NewJSONResponse(map[string]interface{}{
"hits": list,
})
},
))
var AsyncAPI = app.AsyncAPI()
keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
keywordCrawlTask.Queue.Rate = "1/s"
keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
keywordCrawlTask.OnTrigger(middleware.Gate("family"))
keywordCrawlTask.OnMonitor(middleware.Gate("family"))
keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
if data == nil {
return nil, nil
}
var keywords = data.([]pt.IEPGKeyword)
req.Logger.Infof("Crawling %d keywords", len(keywords))
for _, key := range keywords {
list, err := crawlKeyword(req, &key)
if err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
}
return nil, nil
},
))
keywordCrawlTask.ScheduleCron(
"Crawl IEPG keywords",
"every 1 hours",
)
}
func setupAPIOAuth2(app *server.App) {
var API = app.API()
API.GET("/oauth2/clients/",
middleware.EntityAll(entities.OAuth2ClientSettings.Query()))
API.GET("/oauth2/clients/:key.json",
middleware.EntityGet(entities.OAuth2ClientSettings.Get().Cache(true), "key"))
// Create an oauth2.Config object into session and redirect to the oauth2 endpoint.
// /admin/oauth2/callback.html will finally authorize the callback code and store the oauth2.Config into datastore.
API.POST("/oauth2/clients/",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("key").Required()
v.Field("client_id").Required()
v.Field("client_secret").Required()
v.Field("auth_url").Required()
v.Field("token_url").Required()
}),
server.Handler(func(req *wcg.Request) response.Response {
cfg := &oauth2.Config{
ClientID: req.Form("client_id"),
ClientSecret: req.Form("client_secret"),
Endpoint: oauth2.Endpoint{
AuthURL: req.Form("auth_url"),
TokenURL: req.Form("token_url"),
},
Scopes: strings.Split(req.Form("scopes"), ","),
RedirectURL: wcg.AbsoluteURL(
req,
fmt.Sprintf("/api/admin/oauth2/callback/?oauth2_key=%s", req.Form("key")),
),
}
data := wcg.DataBag{}
data.Set("key", req.Form("key"))
data.Set("client_id", cfg.ClientID)
data.Set("client_secret", cfg.ClientSecret)
data.Set("auth_url", cfg.Endpoint.AuthURL)
data.Set("token_url", cfg.Endpoint.TokenURL)
data.Set("scopes", req.Form("scopes"))
data.Set("redirect_url", cfg.RedirectURL)
req.Session.SetData(
fmt.Sprintf("admin.oauth2_%s", req.Form("key")),
data,
)
return response.NewRedirect(cfg.AuthCodeURL("state", oauth2.AccessTypeOffline), response.RedirectSeeOther)
}))
API.DELETE("/oauth2/clients/:client_id.json",
middleware.EntityDelete(entities.OAuth2ClientSettings.Delete(), "client_id"))
API.GET("/oauth2/callback/",
server.Handler(func(req *wcg.Request) response.Response {
key := req.Query("oauth2_key")
code := req.Query("code")
data, ok := req.Session.GetData(fmt.Sprintf("admin.oauth2_%s", key))
if !ok {
return response.NotFound(req)
}
scopes, _ := data.Get("scopes")
cfg := &oauth2.Config{}
cfg.ClientID, _ = data.Get("client_id")
cfg.ClientSecret, _ = data.Get("client_secret")
cfg.Endpoint.AuthURL, _ = data.Get("auth_url")
cfg.Endpoint.TokenURL, _ = data.Get("token_url")
cfg.Scopes = strings.Split(scopes, ",")
cfg.RedirectURL, _ = data.Get("redirect_url")
token, err := cfg.Exchange(gae.NewContext(req), code)
if err != nil {
return response.BadRequest(req, err)
}
settings := models.NewOAuth2ClientSettings(cfg, token)
settings.Key = key
entities.OAuth2ClientSettings.Put().Key(key).MustUpdate(req, settings)
return response.NewRedirect("/admin/oauth2/", response.RedirectSeeOther)
}))
}
func setupAPICrawlerSettings(app *server.App) {
var API = app.API()
var urlValidator = server.Handler(func(req *wcg.Request) response.Response {
_, err := url.Parse(req.Param("url"))
if err != nil {
return response.APINotFound
}
return nil
})
API.GET("/crawlersettings/:type/",
middleware.EntityAll(CrawlerSettings.Query().Filter("Type=", request.Value(func(req *wcg.Request) interface{} {
t, _ := hplink.ParseCrawlerSettingsType(req.Param("type"))
if t == hplink.CrawlerSettingsTypeUnknown {
return entities.FilterValueSkip
}
return t
}))),
)
API.GET("/crawlersettings/ameblo/stats/",
server.Handler(func(req *wcg.Request) response.Response {
type post struct {
URL string `json:"url"`
At time.Time `json:"at"`
}
type stats struct {
URL string `json:"url"`
FirstPost *post `json:"first_post,omitempty"`
LastPost *post `json:"last_post,omitempty"`
TotalPosts int `json:"total_posts"`
CrawledPosts int `json:"crawled_posts"`
ImageFailurePosts int `json:"image_failure_posts"`
}
p := CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
settings := p.Data.([]hplink.CrawlerSettings)
s := make([]stats, len(settings))
if err := iterator.ParallelSlice(settings, func(i int, v *hplink.CrawlerSettings) error {
s[i].URL = v.URL
s[i].TotalPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).MustCount(req)
if s[i].TotalPosts > 0 {
s[i].CrawledPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("IsContentsCrawled=", true).MustCount(req)
s[i].ImageFailurePosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("Images.Height=", 0).MustCount(req)
pf := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("PostAt").Limit(1).MustExecute(req)
pl := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("-PostAt").Limit(1).MustExecute(req)
first := pf.Head().(*hplink.AmebloPost)
last := pl.Head().(*hplink.AmebloPost)
s[i].FirstPost = &post{
URL: first.URL, At: first.PostAt,
}
s[i].LastPost = &post{
URL: last.URL, At: last.PostAt,
}
return nil
}
return nil
}); err != nil {
panic(err)
}
return response.NewJSONResponse(s)
}),
)
API.GET("/crawlersettings/:url.json",
middleware.EntityGet(CrawlerSettings.Get(), "url"),
)
API.PUT("/crawlersettings/:url.json",
urlValidator,
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("artist_key").Required()
}),
middleware.EntityPutOrCreate(
CrawlerSettings.Put(),
"url",
),
)
API.DELETE("/crawlersettings/:url.json",
urlValidator,
middleware.EntityDelete(CrawlerSettings.Delete(), "url"),
)
}
func setupAPIIEPGRecords(app *server.App) {
var API = app.API()
API.GET("/iepg/records/",
middleware.EntityQuery(recentRecordsQuery()))
API.GET("/iepg/records/:id.json",
middleware.EntityGet(IEPG.Get().Cache(true), "id"))
API.POST("/iepg/records/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("program_title").Required()
v.Field("start_at").Required()
v.Field("end_at").Required()
v.Field("category").Required()
v.Field("cid").Required()
v.Field("sid").Required()
}),
middleware.EntityPost(
IEPG.Put().
SetField("Source", pt.IEPGSourceUser).
Cache(_CacheRecentIEPGRecords),
),
)
API.PUT("/iepg/records/:id.json",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("program_title").Required()
v.Field("start_at").Required()
v.Field("end_at").Required()
v.Field("category").Required()
v.Field("cid").Required()
v.Field("sid").Required()
}),
requireIEPGOwner("id"),
middleware.EntityPut(
IEPG.Put().
SetField("Source", pt.IEPGSourceUser).
Cache(_CacheRecentIEPGRecords),
"id",
),
)
API.POST("/iepg/records/:id/notification.json",
middleware.Gate("family"),
getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
var msg string
var configs []pt.MessengerNotification
var err error
t := req.Form("type")
switch t {
case "start":
msg = fmt.Sprintf("[%s] 録画を開始しました。", iepg.ProgramTitle)
configs, err = getMessengerNotificationOptInUsers(req, "OnStart")
break
case "end":
msg = fmt.Sprintf("[%s] 録画を終了しました。", iepg.ProgramTitle)
configs, err = getMessengerNotificationOptInUsers(req, "OnEnd")
break
default:
return response.BadRequest(req, fmt.Errorf("`type` should be specified and must be one of 'start', 'end' (actual=%s)", t))
}
if err != nil {
return response.InternalServerError(req, err)
}
if req.Form("message") != "" {
msg = fmt.Sprintf("%s\n%s", msg, req.Form("message"))
}
for _, cfg := range configs {
client := messenger.NewMessengerClient(req)
client.UserID = cfg.UserID
err = client.SendText(msg)
if err != nil {
req.Logger.Warnf("Failed to send a messenger message: %v", err)
}
}
return response.NewJSONResponse(true)
}),
)
API.DELETE("/iepg/records/:id.json",
middleware.Gate("family"),
requireIEPGOwner("id"),
middleware.EntityDelete(IEPG.Delete().Cache(_CacheRecentIEPGRecords), "id"))
// optin or optout for auto collected IEPGs
API.PUT("/iepg/records/:id/opt-in/",
middleware.Gate("family"),
getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
err := iepg.SetOptIn(req.Form("opt_in") == "1" || req.Form("opt_in") == "true")
if err != nil {
return response.BadRequest(req, fmt.Errorf("Could not configure opt-in field on the user created IEPG"))
}
IEPG.Put().Key(req.Param("id")).Cache(_CacheRecentIEPGRecords).MustUpdate(req, iepg)
return response.NewJSONResponse(map[string]bool{
"ok": true,
"opt_in": iepg.OptIn,
"opt_out": iepg.OptOut,
})
}),
)
}