func setupAPIConfigs(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// Sever Config
API.GET("/configs/",
server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(configs.GetAll(req))
}))
API.GET("/configs/:key.json",
server.Handler(func(req *wcg.Request) response.Response {
val := configs.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
return response.NewJSONResponse(val)
}
}),
)
API.PUT("/configs/:key.json",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("value").Required()
}),
server.Handler(func(req *wcg.Request) response.Response {
val := configs.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
val.Value = req.Form("value")
configs.PutMulti(req, val)
return response.NewJSONResponse(val)
}
}))
}
func setupAPIGates(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// Gate
API.GET("/gates/",
server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(gates.GetAll(req))
}))
API.GET("/gates/:key.json",
server.Handler(func(req *wcg.Request) response.Response {
val := gates.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
return response.NewJSONResponse(val)
}
}))
API.PUT("/gates/:key.json",
middleware.ParseForm(nil),
server.Handler(func(req *wcg.Request) response.Response {
val := gates.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
val.UIDs = req.HTTPRequest().Form["uids"]
gates.PutMulti(req, val)
return response.NewJSONResponse(val)
}
}))
}
func EntityTail(query *entities.Query) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
result := query.MustExecute(req)
return response.NewJSONResponse(result.Tail())
})
}
func EntityPutOrCreate(put *entities.Put, keyParams string) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
return processEntityPut(req, put, keyParams, true)
},
)
}
func EntityDelete(del *entities.Delete, keyParam string) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
del.Key(req.Param(keyParam)).MustCommit(req)
return response.NewJSONResponse(true)
})
}
func ParseForm(configure func(*validators.FormValidator)) server.Handler {
// it's ok to set nil as configure to parse form without validation.
var v *validators.FormValidator
if configure != nil {
v = validators.NewFormValidator()
configure(v)
}
return server.Handler(
func(req *wcg.Request) response.Response {
if m := req.Method(); m != "POST" && m != "PUT" {
return nil
}
if req.Header("content-type") != "application/x-www-form-urlencoded" {
req.Logger.Debugf("Skip FormValidation since the requested content is not a form")
return nil
}
err := req.HTTPRequest().ParseForm()
if err == nil {
req.Logger.Debugf("Request Form: %v", req.HTTPRequest().Form)
} else {
req.Logger.Warnf("ParseForm failed in FormValidaiton process. Triggered ValidationError: %v", err)
return response.BadRequest(req, err)
}
if v == nil {
req.Logger.Debugf("Skip FormValidation since no validator is defined.")
return nil
}
if err := v.Eval(req.HTTPRequest().Form); err != nil {
return response.BadRequest(req, err)
}
return nil
})
}
func Parallel(handlers map[string]server.Handler) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
var status = make(map[string]int)
var data = make(map[string]interface{})
var mutex sync.Mutex
var wg sync.WaitGroup
for k, q := range handlers {
wg.Add(1)
go func(k string, h server.Handler) {
defer wg.Done()
v := h.Handle(req)
if vv, ok := v.(*response.JSONResponse); ok {
mutex.Lock()
defer mutex.Unlock()
status[k] = vv.StatusCode
data[k] = vv.Data
}
}(k, q)
}
wg.Wait()
for k, code := range status {
// return non 2xx response
if code > 0 && (code < 200 || code >= 300) {
return response.NewJSONResponseWithStatus(data[k], code)
}
}
return response.NewJSONResponse(data)
})
}
func setupAPIStats(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// Stats
API.GET("/stats/",
server.Handler(func(req *wcg.Request) response.Response {
stats, _ := runtime.Stats(gae.NewContext(req))
json := map[string]interface{}{
"version": lib.Commit,
"timestamp": lib.Timestamp,
"stats": stats,
"environment": lib.CurrentEnvironment(),
"envvars": lib.GetEnvVars(),
}
if _, dsStats, err := entities.DatastoreStat.Get().Key("total_entities_usage").One(req); err != nil {
json["datastore"] = dsStats
}
// use array response for API convention.
return response.NewJSONResponse(
[](map[string]interface{}){
json,
},
)
}))
}
func getIEPG(id string, f func(req *wcg.Request, iepg *pt.IEPG) response.Response) server.Handler {
return server.Handler(func(req *wcg.Request) response.Response {
_, one := IEPG.Get().Key(req.Param(id)).Cache(true).MustOne(req)
if one == nil {
return response.NotFound(req)
}
return f(req, one.(*pt.IEPG))
})
}
func setupAPIUsers(app *server.App) {
var API = app.API()
API.GET("/me/",
server.Handler(func(req *wcg.Request) response.Response {
if req.User == nil || request.ByGuest(req) {
return response.NewJSONResponse(nil)
}
return response.NewJSONResponse(req.User)
}))
API.POST("/me/gates/",
server.Handler(func(req *wcg.Request) response.Response {
keys := strings.Split(req.Form("keys"), ",")
result := gates.EvalKeys(req, keys...)
return response.NewJSONResponse(result)
}))
}
func EntityGet(get *entities.Get, keyParam string) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
_, one := get.Key(req.Param(keyParam)).MustOne(req)
if one == nil {
return response.APINotFound
}
return response.NewJSONResponse(one)
})
}
func setupAPIConfigs(app *server.App) {
var API = app.API()
API.GET("/configs/:userid/messenger.json",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
server.Handler(
func(req *wcg.Request) response.Response {
_, one := MessengerNotification.Get().Key(req.Param("userid")).UseDefaultIfNil(true).Cache(true).MustOne(req)
return response.NewJSONResponse(one)
},
))
API.PUT("/configs/:userid/messenger.json",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
middleware.ParseForm(nil),
middleware.EntityPutOrCreate(MessengerNotification.Put(), "userid"))
API.POST("/configs/:userid/messenger/notify/",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
server.Handler(
func(req *wcg.Request) response.Response {
content, err := createDailyNotificationContent(req)
if err != nil {
return response.InternalServerError(req, err)
}
err = messenger.NewMessengerClient(req).SendText(content.title + "\n" + strings.Join(content.lines, "\n"))
if err != nil {
req.Logger.Errorf("Failed to notify a message to messenger: %v", err)
}
return response.NewJSONResponse(map[string]interface{}{
"ok": true,
})
},
))
}
func EntityPost(post *entities.Put) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
ent, err := post.Kind().CreateEntityFromForm(req.HTTPRequest().Form)
if err != nil { // ParseError
req.Logger.Warnf("You should use ParseForm middleware to avoid the error in EntityPost: %v", err)
return response.BadRequest(req, ErrInvalidFormParameters)
}
_, ent_ := post.MustUpdate(req, ent)
return response.NewJSONResponseWithStatus(ent_, 201)
})
}
func Test_Parallel(t *testing.T) {
assert := gaetest.NewAssert(t)
router := wcg.NewRouter()
funA := server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(true)
})
funB := server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(false)
})
router.GET("/*", Parallel(map[string]server.Handler{
"a": funA,
"b": funB,
}))
req := ts.GET("/a/")
res := req.RouteTo(router)
var got map[string]bool
assert.HTTPStatus(200, res)
assert.JSONResponse(&got, res)
assert.OK(got["a"])
assert.Not(got["b"])
}
func ValidateQuery(configure func(*validators.FormValidator)) server.Handler {
if configure == nil {
panic(fmt.Errorf("You must pass configure function on ValidateQuery"))
}
v := validators.NewFormValidator()
configure(v)
return server.Handler(
func(req *wcg.Request) response.Response {
if err := v.Eval(req.URL().Query()); err != nil {
return response.BadRequest(req, err)
}
return nil
})
}
func MemberHandler(handlers ...func(*wcg.Request, *hplink.Member) response.Response) server.Handler {
return server.Handler(func(req *wcg.Request) response.Response {
member := findArtistAndMember(req)
if member == nil {
return response.APINotFound
}
for _, h := range handlers {
if resp := h(req, member); resp != nil {
return resp
}
}
return nil
})
}
func EntityStreaming(query *entities.Query, includeKeys bool) server.Handler {
return server.Handler(
func(req *wcg.Request) response.Response {
iter := query.MustRun(req)
pr, pw := io.Pipe()
return response.NewStreamingResponse(&entityStreamer{
req: req,
iter: iter,
pr: pr,
pw: pw,
includeKeys: includeKeys,
})
},
)
}
func setupAPITasks(app *server.App) {
var API = app.API()
API.GET("/tasks/", server.Handler(
middleware.EntityQuery(
entities.AsyncAPITask.Query().Filter(
"UpdatedAt >=",
request.Value(func(req *wcg.Request) interface{} {
return wcg.ParseDateOr(req.Query("since"), lib.Now().Add(-_TaskExpirationDays))
}),
).Filter(
"UpdatedAt <=",
request.Value(func(req *wcg.Request) interface{} {
return wcg.ParseDateOr(req.Query("until"), lib.Now())
}),
),
),
))
}
func ArtistHandler(handlers ...func(*wcg.Request, *hplink.Artist) response.Response) server.Handler {
return server.Handler(func(req *wcg.Request) response.Response {
artistKey := req.Param(paramKeyArtist)
if artistKey == "" {
panic("[BUG] :artist is not found in URL path.")
}
_, ent := entities.Artist.Get().Key(req.Param(paramKeyArtist)).MustOne(req)
if ent == nil {
return response.PageNotFound
}
artist := ent.(*hplink.Artist)
for _, h := range handlers {
if resp := h(req, artist); resp != nil {
return resp
}
}
return nil
})
}
func setupAPIArtist(app *server.App) {
var API = app.API()
API.GET("/",
server.Handler(func(req *wcg.Request) response.Response {
pArtists := entities.Artist.Query().Order("Index").Cache(ckAllArtits).MustExecute(req)
artist := pArtists.Data.([]hplink.Artist)
if req.Query("w") == "members" {
iterator.MustParallelSlice(artist, func(i int, a *hplink.Artist) error {
pMembers := entities.Member.Query().Order("Index").Filter("ArtistKey=", a.Key).Cache(fmt.Sprintf(ckAllMembersTemplate, a.Key)).MustExecute(req)
artist[i].Members = pMembers.Data.([]hplink.Member)
return nil
})
}
return response.NewJSONResponse(artist)
}),
)
API.GET("/:artist/",
middleware.EntityGet(entities.Artist.Get().Cache(true), "artist"))
}
func setupAPIStats(app *server.App) {
var API = app.API()
API.GET("/stats/servers/",
middleware.EntityAll(Server.Query()))
API.GET("/stats/nasdisks/",
middleware.EntityAll(NASDisk.Query()))
API.GET("/stats/servers/:server/:stats/",
server.Handler(
func(req *wcg.Request) response.Response {
key, _ := Server.Get().Key(req.Param("server")).Cache(true).MustOne(req)
if key == nil {
req.Logger.Debugf("server %q is not found.", req.Param("server"))
return response.NotFound(req)
}
var kind *entities.Kind
switch req.Param("stats") {
case "system":
kind = SystemStats
break
case "cpu":
kind = CPUStats
break
case "memory":
kind = MemoryStats
break
case "disk":
kind = DiskStats
break
case "filesystem":
kind = FileSystemStats
break
case "network":
kind = NetworkStats
break
default:
req.Logger.Debugf("metric %q is invalid.", req.Param("stats"))
return response.NotFound(req)
}
return execStatsQuery(req, kind.Query().Ancestor(key))
},
))
API.POST("/stats/metrics/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := SystemMetric.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
collection := home.NewStatsFromMetrics(ent.([]*home.SystemMetric))
if len(collection) == 0 {
return response.BadRequest(req, fmt.Errorf("No stats are collected."))
}
if collection[0].System.ServerName == "" {
return response.BadRequest(req, fmt.Errorf("server_name is missing"))
}
for _, stats := range collection {
key := getServerKey(req, stats.System.ServerName)
id := fmt.Sprintf("%s.%d", key.StringID(), stats.Timestamp.Unix())
stats.System.ID = id
stats.CPU.ID = id
stats.Memory.ID = id
for _, v := range stats.Disks {
v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
}
for _, v := range stats.Networks {
v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
}
SystemStats.Put().Parent(key).MustUpdate(req, stats.System)
CPUStats.Put().Parent(key).MustUpdate(req, stats.CPU)
MemoryStats.Put().Parent(key).MustUpdate(req, stats.Memory)
DiskStats.PutMulti().Parent(key).MustUpdate(req, stats.Disks)
NetworkStats.PutMulti().Parent(key).MustUpdate(req, stats.Networks)
}
return response.NewJSONResponse(true)
},
))
API.POST("/stats/filesystem/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := FileSystemStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
filesystems := ent.([]*home.FileSystemStats)
if filesystems[0].ServerName == "" {
return response.BadRequest(req, fmt.Errorf("server_name is missing"))
}
key := getServerKey(req, filesystems[0].ServerName)
for _, fs := range filesystems {
fs.ID = fmt.Sprintf("%s.%s.%d", fs.ServerName, fs.Name, fs.Timestamp.Unix())
}
FileSystemStats.PutMulti().Parent(key).MustUpdate(req, filesystems)
return response.NewJSONResponse(true)
},
))
API.POST("/stats/smart/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := SMARTStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
smartstats := ent.([]*home.SMARTStats)
if smartstats[0].Serial == "" {
return response.BadRequest(req, fmt.Errorf("serial is missing"))
}
key := getNASDiskKey(req, smartstats[0].Serial)
for _, smart := range smartstats {
smart.ID = fmt.Sprintf("%s.%d", smart.Serial, smart.Timestamp.Unix())
}
SMARTStats.PutMulti().Parent(key).MustUpdate(req, smartstats)
return response.NewJSONResponse(true)
},
))
}
"fmt"
"lib/server"
"lib/server/configs"
"lib/server/response"
"lib/wcg"
)
// SetupHandler is a server.Handler to respond setup HTTP call from a Facebook server
var SetupHandler = server.Handler(func(req *wcg.Request) response.Response {
token := configs.GetValue(req, "facebook_messenger_verify_token")
if token == "" {
return response.NotFound(req)
}
if req.Query("hub.verify_token") != token {
return response.BadRequest(
req,
fmt.Errorf(
"token mismatch: hub.verify_token(%q) != facebook_messenger_verify_token(%q)",
req.Query("hub.verify_token"), token,
),
)
}
return response.NewTextResponse(req.Query("hub.challenge"))
})
// WebhookHandler is a server.Handler to respond setup HTTP call from a Facebook server
var WebHookHandler = server.Handler(func(req *wcg.Request) response.Response {
messages, perr := parseMessage(req.HTTPRequest().Body)
if perr != nil {
req.Logger.Warnf("Server could not parse incomming messenger hook message: %s", perr)
return response.NewJSONResponse(true)
}
func setup20161009Backfill(app *server.App) string {
const batchSize = 800
const cursorKey = "c"
const path = "/tasks/20161009backfill/"
var API = app.API()
var resolveSettingsURL = func(url string, settingsList []hplink.CrawlerSettings) string {
for _, settings := range settingsList {
if strings.HasPrefix(url, settings.URL) {
return settings.URL
}
}
return ""
}
API.POST(
path,
handlers.AdminGate,
server.Handler(func(req *wcg.Request) response.Response {
cursor := req.Query(cursorKey)
logger := wcg.NewLoggerWithPrefix(req, "20161009backfill")
logger.Infof("Start operation (c=%q)", cursor)
p := entities.CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
if p.Len() == 0 {
logger.Infof("No CrawlerSettings found.")
return nil
}
settingsList := p.Data.([]hplink.CrawlerSettings)
q := entities.AmebloPost.Query().Order("-UpdatedAt")
if cursor != "" {
q = q.StartCursor(cursor)
}
iter := q.MustRun(req)
var updates []hplink.AmebloPost
var needMoreIteration = true
for i := 0; i < batchSize; i++ {
_, _post := iter.MustNext()
if _post == nil {
needMoreIteration = false
break
}
post := _post.(*hplink.AmebloPost)
if post.SettingsURL == "" {
logger.Infof("Set %q for SettingsURL on %q", post.SettingsURL, post.URL)
post.SettingsURL = resolveSettingsURL(post.URL, settingsList)
}
updates = append(updates, *post)
}
entities.AmebloPost.PutMulti().DoNotUpdateTimestamp(true).MustUpdate(req, updates)
if needMoreIteration {
_path := fmt.Sprintf("%s?%s", API.Path(path), url.Values{
cursorKey: []string{iter.MustCursorString()},
}.Encode())
if err := server.RunInOpsQueue(req, _path, nil); err != nil {
logger.Errorf("Failed to continue the task: %v", err)
}
}
return response.APIOK
}),
)
return API.Path(path)
}
func setupAPICrawlerSettings(app *server.App) {
var API = app.API()
var urlValidator = server.Handler(func(req *wcg.Request) response.Response {
_, err := url.Parse(req.Param("url"))
if err != nil {
return response.APINotFound
}
return nil
})
API.GET("/crawlersettings/:type/",
middleware.EntityAll(CrawlerSettings.Query().Filter("Type=", request.Value(func(req *wcg.Request) interface{} {
t, _ := hplink.ParseCrawlerSettingsType(req.Param("type"))
if t == hplink.CrawlerSettingsTypeUnknown {
return entities.FilterValueSkip
}
return t
}))),
)
API.GET("/crawlersettings/ameblo/stats/",
server.Handler(func(req *wcg.Request) response.Response {
type post struct {
URL string `json:"url"`
At time.Time `json:"at"`
}
type stats struct {
URL string `json:"url"`
FirstPost *post `json:"first_post,omitempty"`
LastPost *post `json:"last_post,omitempty"`
TotalPosts int `json:"total_posts"`
CrawledPosts int `json:"crawled_posts"`
ImageFailurePosts int `json:"image_failure_posts"`
}
p := CrawlerSettings.Query().Filter("Type=", hplink.CrawlerSettingsTypeAmeblo).MustExecute(req)
settings := p.Data.([]hplink.CrawlerSettings)
s := make([]stats, len(settings))
if err := iterator.ParallelSlice(settings, func(i int, v *hplink.CrawlerSettings) error {
s[i].URL = v.URL
s[i].TotalPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).MustCount(req)
if s[i].TotalPosts > 0 {
s[i].CrawledPosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("IsContentsCrawled=", true).MustCount(req)
s[i].ImageFailurePosts = AmebloPost.Query().Filter("SettingsURL=", v.URL).Filter("Images.Height=", 0).MustCount(req)
pf := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("PostAt").Limit(1).MustExecute(req)
pl := AmebloPost.Query().Filter("SettingsURL=", v.URL).Order("-PostAt").Limit(1).MustExecute(req)
first := pf.Head().(*hplink.AmebloPost)
last := pl.Head().(*hplink.AmebloPost)
s[i].FirstPost = &post{
URL: first.URL, At: first.PostAt,
}
s[i].LastPost = &post{
URL: last.URL, At: last.PostAt,
}
return nil
}
return nil
}); err != nil {
panic(err)
}
return response.NewJSONResponse(s)
}),
)
API.GET("/crawlersettings/:url.json",
middleware.EntityGet(CrawlerSettings.Get(), "url"),
)
API.PUT("/crawlersettings/:url.json",
urlValidator,
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("artist_key").Required()
}),
middleware.EntityPutOrCreate(
CrawlerSettings.Put(),
"url",
),
)
API.DELETE("/crawlersettings/:url.json",
urlValidator,
middleware.EntityDelete(CrawlerSettings.Delete(), "url"),
)
}
func setupAPIDatastore(app *server.App) {
var API = app.API()
if lib.IsOnLocalGAE() {
API.GET("/datastore/cleanup/:kind.json",
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
num := k.DeleteQuery().MustCommit(req)
return response.NewJSONResponse(map[string]int{
"entities": num,
})
}),
)
}
API.GET("/datastore/export/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
// TODO: export with gzip
return middleware.EntityStreaming(
k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit(
wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32),
),
true,
).Handle(req)
}))
API.POST("/datastore/import/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
var line = 0
var total = 0
var errors = make([]*entityImportError, 0)
var list = make([]interface{}, 0, 100)
var keys = make([]*datastore.Key, 0, 100)
var buff []byte
var err error
reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096)
for err != io.EOF {
buff, err = reader.ReadBytes('\n')
line += 1
if err != nil && err != io.EOF {
return response.InternalServerError(req, err)
}
buff = bytes.TrimSpace(buff)
if len(buff) == 0 { // skip empty row
continue
}
var row entityImportRow
if err2 := json.Unmarshal(buff, &row); err2 != nil {
errors = append(errors, &entityImportError{
Line: line,
Error: fmt.Sprintf("%v - (row: %q)", err2, buff),
})
continue
}
ent, err2 := k.NewEntityFromJSON([]byte(row.Entity))
if err2 != nil {
}
total = total + 1
keys = append(keys, buildKey(req, row.Key))
list = append(list, ent)
if len(list) == 100 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
list = make([]interface{}, 0, 100)
keys = make([]*datastore.Key, 0, 100)
}
}
if len(list) > 0 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
}
return response.NewJSONResponse(&entityImportResult{
Total: total,
Lines: line - 1,
Errors: errors,
})
}))
}
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
server.Handler(
func(req *wcg.Request) response.Response {
content, err := createDailyNotificationContent(req)
if err != nil {
return response.InternalServerError(req, err)
}
err = messenger.NewMessengerClient(req).SendText(content.title + "\n" + strings.Join(content.lines, "\n"))
if err != nil {
req.Logger.Errorf("Failed to notify a message to messenger: %v", err)
}
return response.NewJSONResponse(map[string]interface{}{
"ok": true,
})
},
))
}
var requireMessengeOptIn = middleware.RequireMessengerOptIn("")
var validateUserID = server.Handler(func(req *wcg.Request) response.Response {
if req.Param("userid") != req.User.ID() {
req.Logger.Warnf("Invalid userid (Requested: %s, Actual: %s)", req.Param("userid"), req.User.ID())
return response.Forbidden(req)
}
return nil
})
func setupAPIOAuth2(app *server.App) {
var API = app.API()
API.GET("/oauth2/clients/",
middleware.EntityAll(entities.OAuth2ClientSettings.Query()))
API.GET("/oauth2/clients/:key.json",
middleware.EntityGet(entities.OAuth2ClientSettings.Get().Cache(true), "key"))
// Create an oauth2.Config object into session and redirect to the oauth2 endpoint.
// /admin/oauth2/callback.html will finally authorize the callback code and store the oauth2.Config into datastore.
API.POST("/oauth2/clients/",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("key").Required()
v.Field("client_id").Required()
v.Field("client_secret").Required()
v.Field("auth_url").Required()
v.Field("token_url").Required()
}),
server.Handler(func(req *wcg.Request) response.Response {
cfg := &oauth2.Config{
ClientID: req.Form("client_id"),
ClientSecret: req.Form("client_secret"),
Endpoint: oauth2.Endpoint{
AuthURL: req.Form("auth_url"),
TokenURL: req.Form("token_url"),
},
Scopes: strings.Split(req.Form("scopes"), ","),
RedirectURL: wcg.AbsoluteURL(
req,
fmt.Sprintf("/api/admin/oauth2/callback/?oauth2_key=%s", req.Form("key")),
),
}
data := wcg.DataBag{}
data.Set("key", req.Form("key"))
data.Set("client_id", cfg.ClientID)
data.Set("client_secret", cfg.ClientSecret)
data.Set("auth_url", cfg.Endpoint.AuthURL)
data.Set("token_url", cfg.Endpoint.TokenURL)
data.Set("scopes", req.Form("scopes"))
data.Set("redirect_url", cfg.RedirectURL)
req.Session.SetData(
fmt.Sprintf("admin.oauth2_%s", req.Form("key")),
data,
)
return response.NewRedirect(cfg.AuthCodeURL("state", oauth2.AccessTypeOffline), response.RedirectSeeOther)
}))
API.DELETE("/oauth2/clients/:client_id.json",
middleware.EntityDelete(entities.OAuth2ClientSettings.Delete(), "client_id"))
API.GET("/oauth2/callback/",
server.Handler(func(req *wcg.Request) response.Response {
key := req.Query("oauth2_key")
code := req.Query("code")
data, ok := req.Session.GetData(fmt.Sprintf("admin.oauth2_%s", key))
if !ok {
return response.NotFound(req)
}
scopes, _ := data.Get("scopes")
cfg := &oauth2.Config{}
cfg.ClientID, _ = data.Get("client_id")
cfg.ClientSecret, _ = data.Get("client_secret")
cfg.Endpoint.AuthURL, _ = data.Get("auth_url")
cfg.Endpoint.TokenURL, _ = data.Get("token_url")
cfg.Scopes = strings.Split(scopes, ",")
cfg.RedirectURL, _ = data.Get("redirect_url")
token, err := cfg.Exchange(gae.NewContext(req), code)
if err != nil {
return response.BadRequest(req, err)
}
settings := models.NewOAuth2ClientSettings(cfg, token)
settings.Key = key
entities.OAuth2ClientSettings.Put().Key(key).MustUpdate(req, settings)
return response.NewRedirect("/admin/oauth2/", response.RedirectSeeOther)
}))
}
func setupAPIIEPGKeywords(app *server.App) {
var API = app.API()
API.GET("/iepg/keywords/",
middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))
API.POST("/iepg/keywords/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.Field("category").Required()
v.IntField("scope").Required()
}),
middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))
API.DELETE("/iepg/keywords/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))
API.POST("/iepg/keywords/preview/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.IntField("scope").Required()
}),
server.Handler(
func(req *wcg.Request) response.Response {
list, err := crawlKeyword(req, &pt.IEPGKeyword{
Keyword: req.Form("keyword"),
Scope: pt.ParseIEPGCrawlerScope(req.Form("scope")),
})
if err != nil {
return response.InternalServerError(req, err)
}
return response.NewJSONResponse(map[string]interface{}{
"hits": list,
})
},
))
var AsyncAPI = app.AsyncAPI()
keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
keywordCrawlTask.Queue.Rate = "1/s"
keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
keywordCrawlTask.OnTrigger(middleware.Gate("family"))
keywordCrawlTask.OnMonitor(middleware.Gate("family"))
keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
if data == nil {
return nil, nil
}
var keywords = data.([]pt.IEPGKeyword)
req.Logger.Infof("Crawling %d keywords", len(keywords))
for _, key := range keywords {
list, err := crawlKeyword(req, &key)
if err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
}
return nil, nil
},
))
keywordCrawlTask.ScheduleCron(
"Crawl IEPG keywords",
"every 1 hours",
)
}