func setupAPIConfigs(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// Sever Config
API.GET("/configs/",
server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(configs.GetAll(req))
}))
API.GET("/configs/:key.json",
server.Handler(func(req *wcg.Request) response.Response {
val := configs.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
return response.NewJSONResponse(val)
}
}),
)
API.PUT("/configs/:key.json",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("value").Required()
}),
server.Handler(func(req *wcg.Request) response.Response {
val := configs.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
val.Value = req.Form("value")
configs.PutMulti(req, val)
return response.NewJSONResponse(val)
}
}))
}
func setupAPIGates(app *server.App) {
var API = app.API()
// --------------------------------------------------------------------------
// Gate
API.GET("/gates/",
server.Handler(func(req *wcg.Request) response.Response {
return response.NewJSONResponse(gates.GetAll(req))
}))
API.GET("/gates/:key.json",
server.Handler(func(req *wcg.Request) response.Response {
val := gates.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
return response.NewJSONResponse(val)
}
}))
API.PUT("/gates/:key.json",
middleware.ParseForm(nil),
server.Handler(func(req *wcg.Request) response.Response {
val := gates.Get(req, req.Param("key"))
if val == nil {
return response.NotFound(req)
} else {
val.UIDs = req.HTTPRequest().Form["uids"]
gates.PutMulti(req, val)
return response.NewJSONResponse(val)
}
}))
}
func getIEPG(id string, f func(req *wcg.Request, iepg *pt.IEPG) response.Response) server.Handler {
return server.Handler(func(req *wcg.Request) response.Response {
_, one := IEPG.Get().Key(req.Param(id)).Cache(true).MustOne(req)
if one == nil {
return response.NotFound(req)
}
return f(req, one.(*pt.IEPG))
})
}
// NewInstance creates *Instance
func NewInstance() *Instance {
instance := &Instance{
Server: gae.NewServer(),
}
instance.Routes().Before(wcg.NewNamedHandler("init", func(res *wcg.Response, req *wcg.Request) {
instance.once.Do(func() {
req.Logger.Infof("----------- Server initialization ----------- ")
if instance.Init != nil {
instance.Init(req)
}
req.Logger.Infof("----------- Apps initialization ----------- ")
for _, app := range instance.apps {
if app.Init != nil {
app.Init(req)
}
}
req.Logger.Infof("----------- All init processes finished ----------- ")
})
}))
auth.Prepare(instance.Routes(), func() {
instance.Routes().GET("/cron.yaml", wcg.NewNamedHandler("cron.yaml", func(res *wcg.Response, req *wcg.Request) {
res.Header().Add("Content-Type", "text/yaml")
res.WriteString("cron:\n")
for _, app := range instance.apps {
res.WriteString(app.Cron().ToYAML())
}
}))
instance.Routes().GET("/queue.yaml", wcg.NewNamedHandler("queue.yaml", func(res *wcg.Response, req *wcg.Request) {
res.Header().Add("Content-Type", "text/yaml")
res.WriteString("queue:\n")
for _, app := range instance.apps {
res.WriteString(app.Queue().ToYAML())
}
}))
})
instance.Routes().NotFound = Handler(func(req *wcg.Request) response.Response {
return response.NotFound(req)
})
return instance
}
package messenger
import (
"fmt"
"lib/server"
"lib/server/configs"
"lib/server/response"
"lib/wcg"
)
// SetupHandler is a server.Handler to respond setup HTTP call from a Facebook server
var SetupHandler = server.Handler(func(req *wcg.Request) response.Response {
token := configs.GetValue(req, "facebook_messenger_verify_token")
if token == "" {
return response.NotFound(req)
}
if req.Query("hub.verify_token") != token {
return response.BadRequest(
req,
fmt.Errorf(
"token mismatch: hub.verify_token(%q) != facebook_messenger_verify_token(%q)",
req.Query("hub.verify_token"), token,
),
)
}
return response.NewTextResponse(req.Query("hub.challenge"))
})
// WebhookHandler is a server.Handler to respond setup HTTP call from a Facebook server
var WebHookHandler = server.Handler(func(req *wcg.Request) response.Response {
messages, perr := parseMessage(req.HTTPRequest().Body)
func setupAPIDatastore(app *server.App) {
var API = app.API()
if lib.IsOnLocalGAE() {
API.GET("/datastore/cleanup/:kind.json",
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
num := k.DeleteQuery().MustCommit(req)
return response.NewJSONResponse(map[string]int{
"entities": num,
})
}),
)
}
API.GET("/datastore/export/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
// TODO: export with gzip
return middleware.EntityStreaming(
k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit(
wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32),
),
true,
).Handle(req)
}))
API.POST("/datastore/import/:kind.json",
middleware.APITokenOnly(),
server.Handler(func(req *wcg.Request) response.Response {
k := entities.FindKind(req.Param("kind"), req.Query("ns"))
if k == nil {
return response.NotFound(req)
}
var line = 0
var total = 0
var errors = make([]*entityImportError, 0)
var list = make([]interface{}, 0, 100)
var keys = make([]*datastore.Key, 0, 100)
var buff []byte
var err error
reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096)
for err != io.EOF {
buff, err = reader.ReadBytes('\n')
line += 1
if err != nil && err != io.EOF {
return response.InternalServerError(req, err)
}
buff = bytes.TrimSpace(buff)
if len(buff) == 0 { // skip empty row
continue
}
var row entityImportRow
if err2 := json.Unmarshal(buff, &row); err2 != nil {
errors = append(errors, &entityImportError{
Line: line,
Error: fmt.Sprintf("%v - (row: %q)", err2, buff),
})
continue
}
ent, err2 := k.NewEntityFromJSON([]byte(row.Entity))
if err2 != nil {
}
total = total + 1
keys = append(keys, buildKey(req, row.Key))
list = append(list, ent)
if len(list) == 100 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
list = make([]interface{}, 0, 100)
keys = make([]*datastore.Key, 0, 100)
}
}
if len(list) > 0 {
k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
}
return response.NewJSONResponse(&entityImportResult{
Total: total,
Lines: line - 1,
Errors: errors,
})
}))
}
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
if !strings.HasSuffix(path, "/") {
panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
}
api := helper.app.API()
// /path/to/something/ => api-path-to-something
p := api.Path(path)
name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
name = strings.Replace(name, "/", "-", -1) // replace '/' with '-'
config := &AsyncAPIConfig{
Queue: helper.app.Queue().DefinePush(name),
app: helper.app,
path: path,
}
// Endpoint that trigger the task.
api.POST(path, Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
req.Logger.Debugf("Async Task is not triggered.")
return resp
}
}
task := &models.AsyncAPITask{}
task.ID = wcg.NewUUID()
task.Path = path
task.Query = req.HTTPRequest().URL.RawQuery
task.Status = models.AsyncAPIStatusReady
entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
// Push a task
taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
return response.InternalServerError(req, err)
}
req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
return response.NewJSONResponseWithStatus(
AsyncTaskTriggerResponse{task.ID},
201,
)
},
))
// Endpoint that executes the task in background
// This endpoint must be called by PushQueue
api.POST(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
if !request.IsTask(req) {
req.Logger.Warnf(
"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
req.Logger.Warnf(
"Task %q not found: UserAgent=%s, IP=%s",
req.Param("taskid"),
req.HTTPRequest().UserAgent(),
req.HTTPRequest().RemoteAddr,
)
return response.APIOK
}
task := one.(*models.AsyncAPITask)
logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
logger.Warnf("Task is not ready: Status=%s", task.Status)
return response.APIOK
}
if task.Status == models.AsyncAPIStatusReady {
task.StartAt = lib.Now()
task.Status = models.AsyncAPIStatusRunning
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
}
var err error
var progress *models.AsyncAPITaskProgress
var resp response.Response
func() {
defer func() {
if x := recover(); x != nil {
logger.Errorf("Unhandle error recovered from the task: %v", x)
err = fmt.Errorf("%v", x)
}
}()
progress, err = config.processHandler(req, task)
}()
if progress != nil {
task.Progress = append(task.Progress, *progress)
}
if progress != nil && progress.Next != nil {
// update the entity (Progress field)
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
// Push a next request into the queue
// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
// If the code cannot push task for the next request, we should stop the task as failure.
logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
if !req.IsTest() {
err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
}
if err == nil {
return response.NewJSONResponse(progress.Next)
}
// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
}
// finished the task
task.FinishAt = lib.Now()
tt := task.FinishAt.Sub(task.StartAt)
if err == nil {
task.Status = models.AsyncAPIStatusSuccess
logger.Infof("Task finished successfully (time: %s)", tt.String())
resp = response.APIOK
} else {
task.Error = err.Error()
task.Status = models.AsyncAPIStatusFailure
logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
resp = response.APIInternalServerError
}
entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
return resp
},
))
// Endpoint that returns the task status.
api.GET(path+":taskid.json", Handler(
func(req *wcg.Request) response.Response {
for _, h := range config.triggerHandlers {
resp := h.Handle(req)
if resp != nil {
return resp
}
}
_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
if one == nil {
return response.NotFound(req)
}
task := one.(*models.AsyncAPITask)
return response.NewJSONResponse(
AsyncTaskMonitorResponse{
ID: task.ID,
Status: task.Status,
Progress: task.LastProgress(),
},
)
},
))
return config
}
func setupAPIOAuth2(app *server.App) {
var API = app.API()
API.GET("/oauth2/clients/",
middleware.EntityAll(entities.OAuth2ClientSettings.Query()))
API.GET("/oauth2/clients/:key.json",
middleware.EntityGet(entities.OAuth2ClientSettings.Get().Cache(true), "key"))
// Create an oauth2.Config object into session and redirect to the oauth2 endpoint.
// /admin/oauth2/callback.html will finally authorize the callback code and store the oauth2.Config into datastore.
API.POST("/oauth2/clients/",
middleware.ParseForm(func(v *validators.FormValidator) {
v.Field("key").Required()
v.Field("client_id").Required()
v.Field("client_secret").Required()
v.Field("auth_url").Required()
v.Field("token_url").Required()
}),
server.Handler(func(req *wcg.Request) response.Response {
cfg := &oauth2.Config{
ClientID: req.Form("client_id"),
ClientSecret: req.Form("client_secret"),
Endpoint: oauth2.Endpoint{
AuthURL: req.Form("auth_url"),
TokenURL: req.Form("token_url"),
},
Scopes: strings.Split(req.Form("scopes"), ","),
RedirectURL: wcg.AbsoluteURL(
req,
fmt.Sprintf("/api/admin/oauth2/callback/?oauth2_key=%s", req.Form("key")),
),
}
data := wcg.DataBag{}
data.Set("key", req.Form("key"))
data.Set("client_id", cfg.ClientID)
data.Set("client_secret", cfg.ClientSecret)
data.Set("auth_url", cfg.Endpoint.AuthURL)
data.Set("token_url", cfg.Endpoint.TokenURL)
data.Set("scopes", req.Form("scopes"))
data.Set("redirect_url", cfg.RedirectURL)
req.Session.SetData(
fmt.Sprintf("admin.oauth2_%s", req.Form("key")),
data,
)
return response.NewRedirect(cfg.AuthCodeURL("state", oauth2.AccessTypeOffline), response.RedirectSeeOther)
}))
API.DELETE("/oauth2/clients/:client_id.json",
middleware.EntityDelete(entities.OAuth2ClientSettings.Delete(), "client_id"))
API.GET("/oauth2/callback/",
server.Handler(func(req *wcg.Request) response.Response {
key := req.Query("oauth2_key")
code := req.Query("code")
data, ok := req.Session.GetData(fmt.Sprintf("admin.oauth2_%s", key))
if !ok {
return response.NotFound(req)
}
scopes, _ := data.Get("scopes")
cfg := &oauth2.Config{}
cfg.ClientID, _ = data.Get("client_id")
cfg.ClientSecret, _ = data.Get("client_secret")
cfg.Endpoint.AuthURL, _ = data.Get("auth_url")
cfg.Endpoint.TokenURL, _ = data.Get("token_url")
cfg.Scopes = strings.Split(scopes, ",")
cfg.RedirectURL, _ = data.Get("redirect_url")
token, err := cfg.Exchange(gae.NewContext(req), code)
if err != nil {
return response.BadRequest(req, err)
}
settings := models.NewOAuth2ClientSettings(cfg, token)
settings.Key = key
entities.OAuth2ClientSettings.Put().Key(key).MustUpdate(req, settings)
return response.NewRedirect("/admin/oauth2/", response.RedirectSeeOther)
}))
}
func setupAPIStats(app *server.App) {
var API = app.API()
API.GET("/stats/servers/",
middleware.EntityAll(Server.Query()))
API.GET("/stats/nasdisks/",
middleware.EntityAll(NASDisk.Query()))
API.GET("/stats/servers/:server/:stats/",
server.Handler(
func(req *wcg.Request) response.Response {
key, _ := Server.Get().Key(req.Param("server")).Cache(true).MustOne(req)
if key == nil {
req.Logger.Debugf("server %q is not found.", req.Param("server"))
return response.NotFound(req)
}
var kind *entities.Kind
switch req.Param("stats") {
case "system":
kind = SystemStats
break
case "cpu":
kind = CPUStats
break
case "memory":
kind = MemoryStats
break
case "disk":
kind = DiskStats
break
case "filesystem":
kind = FileSystemStats
break
case "network":
kind = NetworkStats
break
default:
req.Logger.Debugf("metric %q is invalid.", req.Param("stats"))
return response.NotFound(req)
}
return execStatsQuery(req, kind.Query().Ancestor(key))
},
))
API.POST("/stats/metrics/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := SystemMetric.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
collection := home.NewStatsFromMetrics(ent.([]*home.SystemMetric))
if len(collection) == 0 {
return response.BadRequest(req, fmt.Errorf("No stats are collected."))
}
if collection[0].System.ServerName == "" {
return response.BadRequest(req, fmt.Errorf("server_name is missing"))
}
for _, stats := range collection {
key := getServerKey(req, stats.System.ServerName)
id := fmt.Sprintf("%s.%d", key.StringID(), stats.Timestamp.Unix())
stats.System.ID = id
stats.CPU.ID = id
stats.Memory.ID = id
for _, v := range stats.Disks {
v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
}
for _, v := range stats.Networks {
v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
}
SystemStats.Put().Parent(key).MustUpdate(req, stats.System)
CPUStats.Put().Parent(key).MustUpdate(req, stats.CPU)
MemoryStats.Put().Parent(key).MustUpdate(req, stats.Memory)
DiskStats.PutMulti().Parent(key).MustUpdate(req, stats.Disks)
NetworkStats.PutMulti().Parent(key).MustUpdate(req, stats.Networks)
}
return response.NewJSONResponse(true)
},
))
API.POST("/stats/filesystem/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := FileSystemStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
filesystems := ent.([]*home.FileSystemStats)
if filesystems[0].ServerName == "" {
return response.BadRequest(req, fmt.Errorf("server_name is missing"))
}
key := getServerKey(req, filesystems[0].ServerName)
for _, fs := range filesystems {
fs.ID = fmt.Sprintf("%s.%s.%d", fs.ServerName, fs.Name, fs.Timestamp.Unix())
}
FileSystemStats.PutMulti().Parent(key).MustUpdate(req, filesystems)
return response.NewJSONResponse(true)
},
))
API.POST("/stats/smart/",
middleware.APITokenOnly(),
server.Handler(
func(req *wcg.Request) response.Response {
ent, err := SMARTStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
if err != nil {
return response.BadRequest(req, err)
}
smartstats := ent.([]*home.SMARTStats)
if smartstats[0].Serial == "" {
return response.BadRequest(req, fmt.Errorf("serial is missing"))
}
key := getNASDiskKey(req, smartstats[0].Serial)
for _, smart := range smartstats {
smart.ID = fmt.Sprintf("%s.%d", smart.Serial, smart.Timestamp.Unix())
}
SMARTStats.PutMulti().Parent(key).MustUpdate(req, smartstats)
return response.NewJSONResponse(true)
},
))
}