func processEntityPut(req *wcg.Request, put *entities.Put, keyParams string, create bool) response.Response {
key := req.Param(keyParams)
_, ent := put.Kind().Get().Key(key).UseDefaultIfNil(create).MustOne(req)
if ent == nil {
return response.APINotFound
}
if req.HTTPRequest().Form == nil {
req.Logger.Warnf("You should use ParseForm middleware to avoid the error in EntityPut|EntityPutOrCreate")
return response.InternalServerError(req, ErrInvalidFormParameters).(response.Response)
}
err := put.Kind().UpdateEntityFromForm(ent, req.HTTPRequest().Form)
if err != nil {
return response.BadRequest(req, ErrInvalidFormParameters)
}
_, ent_ := put.Key(key).MustUpdate(req, ent)
return response.NewJSONResponseWithStatus(ent_, 200)
}
func setupAPIConfigs(app *server.App) {
var API = app.API()
API.GET("/configs/:userid/messenger.json",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
server.Handler(
func(req *wcg.Request) response.Response {
_, one := MessengerNotification.Get().Key(req.Param("userid")).UseDefaultIfNil(true).Cache(true).MustOne(req)
return response.NewJSONResponse(one)
},
))
API.PUT("/configs/:userid/messenger.json",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
middleware.ParseForm(nil),
middleware.EntityPutOrCreate(MessengerNotification.Put(), "userid"))
API.POST("/configs/:userid/messenger/notify/",
middleware.Gate("family"),
validateUserID,
requireMessengeOptIn,
server.Handler(
func(req *wcg.Request) response.Response {
content, err := createDailyNotificationContent(req)
if err != nil {
return response.InternalServerError(req, err)
}
err = messenger.NewMessengerClient(req).SendText(content.title + "\n" + strings.Join(content.lines, "\n"))
if err != nil {
req.Logger.Errorf("Failed to notify a message to messenger: %v", err)
}
return response.NewJSONResponse(map[string]interface{}{
"ok": true,
})
},
))
}
func setupAPIDatastore(app *server.App) {
var API = app.API()
if lib.IsOnLocalGAE() {
API.GET("/datastore/cleanup/:kind.json",
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
num := k.DeleteQuery().MustCommit(req)
return response.NewJSONResponse(map[string]int{
"entities": num,
})
}),
)
}
API.GET("/datastore/export/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
// TODO: export with gzip
return middleware.EntityStreaming(
k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit(
wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32),
),
true,
).Handle(req)
}))
API.POST("/datastore/import/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
var line = 0
var total = 0
var errors = make([]*entityImportError, 0)
var list = make([]interface{}, 0, 100)
var keys = make([]*datastore.Key, 0, 100)
var buff []byte
var err error
reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096)
for err != io.EOF {
buff, err = reader.ReadBytes('\n')
line += 1
if err != nil && err != io.EOF {
return response.InternalServerError(req, err)
}
buff = bytes.TrimSpace(buff)
if len(buff) == 0 { // skip empty row
continue
}
var row entityImportRow
if err2 := json.Unmarshal(buff, &row); err2 != nil {
errors = append(errors, &entityImportError{
Line: line,
Error: fmt.Sprintf("%v - (row: %q)", err2, buff),
})
continue
}
ent, err2 := k.NewEntityFromJSON([]byte(row.Entity))
if err2 != nil {
}
total = total + 1
keys = append(keys, buildKey(req, row.Key))
list = append(list, ent)
if len(list) == 100 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
list = make([]interface{}, 0, 100)
keys = make([]*datastore.Key, 0, 100)
}
}
if len(list) > 0 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
}
return response.NewJSONResponse(&entityImportResult{
Total: total,
Lines: line - 1,
Errors: errors,
})
}))
}
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
if !strings.HasSuffix(path, "/") {
panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
}
api := helper.app.API()
// /path/to/something/ => api-path-to-something
p := api.Path(path)
name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
name = strings.Replace(name, "/", "-", -1) // replace '/' with '-'
config := &AsyncAPIConfig{
Queue: helper.app.Queue().DefinePush(name),
app: helper.app,
path: path,
}
// Endpoint that trigger the task.
api.POST(path, Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
req.Logger.Debugf("Async Task is not triggered.")
return resp
}
}
task := &models.AsyncAPITask{}
task.ID = wcg.NewUUID()
task.Path = path
task.Query = req.HTTPRequest().URL.RawQuery
task.Status = models.AsyncAPIStatusReady
entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
// Push a task
taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
return response.InternalServerError(req, err)
}
req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
return response.NewJSONResponseWithStatus(
AsyncTaskTriggerResponse{task.ID},
201,
)
},
))
// Endpoint that executes the task in background
// This endpoint must be called by PushQueue
api.POST(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
if !request.IsTask(req) {
req.Logger.Warnf(
"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
req.Logger.Warnf(
"Task %q not found: UserAgent=%s, IP=%s",
req.Param("taskid"),
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
task := one.(*models.AsyncAPITask)
logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
logger.Warnf("Task is not ready: Status=%s", task.Status)
return response.APIOK
}
if task.Status == models.AsyncAPIStatusReady {
task.StartAt = lib.Now()
task.Status = models.AsyncAPIStatusRunning
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
}
var err error
var progress *models.AsyncAPITaskProgress
var resp response.Response
func() {
defer func() {
if x := recover(); x != nil {
logger.Errorf("Unhandle error recovered from the task: %v", x)
err = fmt.Errorf("%v", x)
}
}()
progress, err = config.processHandler(req, task)
}()
if progress != nil {
task.Progress = append(task.Progress, *progress)
}
if progress != nil && progress.Next != nil {
// update the entity (Progress field)
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
// Push a next request into the queue
// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
// If the code cannot push task for the next request, we should stop the task as failure.
logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
if !req.IsTest() {
err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
}
if err == nil {
return response.NewJSONResponse(progress.Next)
}
// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
}
// finished the task
task.FinishAt = lib.Now()
tt := task.FinishAt.Sub(task.StartAt)
if err == nil {
task.Status = models.AsyncAPIStatusSuccess
logger.Infof("Task finished successfully (time: %s)", tt.String())
resp = response.APIOK
} else {
task.Error = err.Error()
task.Status = models.AsyncAPIStatusFailure
logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
resp = response.APIInternalServerError
}
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
return resp
},
))
// Endpoint that returns the task status.
api.GET(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
return resp
}
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
return response.NotFound(req)
}
task := one.(*models.AsyncAPITask)
return response.NewJSONResponse(
AsyncTaskMonitorResponse{
ID: task.ID,
Status: task.Status,
Progress: task.LastProgress(),
},
)
},
))
return config
}
func setupAPIIEPGKeywords(app *server.App) {
var API = app.API()
API.GET("/iepg/keywords/",
middleware.EntityAll(IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords)))
API.POST("/iepg/keywords/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.Field("category").Required()
v.IntField("scope").Required()
}),
middleware.EntityPost(IEPGKeyword.Put().Cache(_CacheAllIEPGKeywords)))
API.DELETE("/iepg/keywords/:id.json",
middleware.Gate("family"),
middleware.EntityDelete(IEPGKeyword.Delete().Cache(_CacheAllIEPGKeywords), "id"))
API.POST("/iepg/keywords/preview/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("keyword").Required()
v.IntField("scope").Required()
}),
server.Handler(
func(req *wcg.Request) response.Response {
list, err := crawlKeyword(req, &pt.IEPGKeyword{
Keyword: req.Form("keyword"),
Scope: pt.ParseIEPGCrawlerScope(req.Form("scope")),
})
if err != nil {
return response.InternalServerError(req, err)
}
return response.NewJSONResponse(map[string]interface{}{
"hits": list,
})
},
))
var AsyncAPI = app.AsyncAPI()
keywordCrawlTask := AsyncAPI.Define("/iepg/keywords/task/")
keywordCrawlTask.Queue.Rate = "1/s"
keywordCrawlTask.Queue.MaxConcurrentRequests = "1"
keywordCrawlTask.OnTrigger(middleware.Gate("family"))
keywordCrawlTask.OnMonitor(middleware.Gate("family"))
keywordCrawlTask.OnProcess(server.AsyncTaskHandler(
func(req *wcg.Request, t *models.AsyncAPITask) (*models.AsyncAPITaskProgress, error) {
data := IEPGKeyword.Query().Cache(_CacheAllIEPGKeywords).MustExecute(req).Data
if data == nil {
return nil, nil
}
var keywords = data.([]pt.IEPGKeyword)
req.Logger.Infof("Crawling %d keywords", len(keywords))
for _, key := range keywords {
list, err := crawlKeyword(req, &key)
if err != nil {
return nil, err
}
IEPG.PutMulti().Cache(_CacheAllIEPGKeywords).MustUpdate(req, list)
}
return nil, nil
},
))
keywordCrawlTask.ScheduleCron(
"Crawl IEPG keywords",
"every 1 hours",
)
}
func setupAPIIEPGRecords(app *server.App) {
var API = app.API()
API.GET("/iepg/records/",
middleware.EntityQuery(recentRecordsQuery()))
API.GET("/iepg/records/:id.json",
middleware.EntityGet(IEPG.Get().Cache(true), "id"))
API.POST("/iepg/records/",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("program_title").Required()
v.Field("start_at").Required()
v.Field("end_at").Required()
v.Field("category").Required()
v.Field("cid").Required()
v.Field("sid").Required()
}),
middleware.EntityPost(
IEPG.Put().
SetField("Source", pt.IEPGSourceUser).
Cache(_CacheRecentIEPGRecords),
),
)
API.PUT("/iepg/records/:id.json",
middleware.Gate("family"),
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("program_title").Required()
v.Field("start_at").Required()
v.Field("end_at").Required()
v.Field("category").Required()
v.Field("cid").Required()
v.Field("sid").Required()
}),
requireIEPGOwner("id"),
middleware.EntityPut(
IEPG.Put().
SetField("Source", pt.IEPGSourceUser).
Cache(_CacheRecentIEPGRecords),
"id",
),
)
API.POST("/iepg/records/:id/notification.json",
middleware.Gate("family"),
getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
var msg string
var configs []pt.MessengerNotification
var err error
t := req.Form("type")
switch t {
case "start":
msg = fmt.Sprintf("[%s] 録画を開始しました。", iepg.ProgramTitle)
configs, err = getMessengerNotificationOptInUsers(req, "OnStart")
break
case "end":
msg = fmt.Sprintf("[%s] 録画を終了しました。", iepg.ProgramTitle)
configs, err = getMessengerNotificationOptInUsers(req, "OnEnd")
break
default:
return response.BadRequest(req, fmt.Errorf("`type` should be specified and must be one of 'start', 'end' (actual=%s)", t))
}
if err != nil {
return response.InternalServerError(req, err)
}
if req.Form("message") != "" {
msg = fmt.Sprintf("%s\n%s", msg, req.Form("message"))
}
for _, cfg := range configs {
client := messenger.NewMessengerClient(req)
client.UserID = cfg.UserID
err = client.SendText(msg)
if err != nil {
req.Logger.Warnf("Failed to send a messenger message: %v", err)
}
}
return response.NewJSONResponse(true)
}),
)
API.DELETE("/iepg/records/:id.json",
middleware.Gate("family"),
requireIEPGOwner("id"),
middleware.EntityDelete(IEPG.Delete().Cache(_CacheRecentIEPGRecords), "id"))
// optin or optout for auto collected IEPGs
API.PUT("/iepg/records/:id/opt-in/",
middleware.Gate("family"),
getIEPG("id", func(req *wcg.Request, iepg *pt.IEPG) response.Response {
err := iepg.SetOptIn(req.Form("opt_in") == "1" || req.Form("opt_in") == "true")
if err != nil {
return response.BadRequest(req, fmt.Errorf("Could not configure opt-in field on the user created IEPG"))
}
IEPG.Put().Key(req.Param("id")).Cache(_CacheRecentIEPGRecords).MustUpdate(req, iepg)
return response.NewJSONResponse(map[string]bool{
"ok": true,
"opt_in": iepg.OptIn,
"opt_out": iepg.OptOut,
})
}),
)
}