func setupAPIConfigs(app *server.App) { var API = app.API() // -------------------------------------------------------------------------- // Sever Config API.GET("/configs/", server.Handler(func(req *wcg.Request) response.Response { return response.NewJSONResponse(configs.GetAll(req)) })) API.GET("/configs/:key.json", server.Handler(func(req *wcg.Request) response.Response { val := configs.Get(req, req.Param("key")) if val == nil { return response.NotFound(req) } else { return response.NewJSONResponse(val) } }), ) API.PUT("/configs/:key.json", middleware.ParseForm(func(v *validators.FormValidator) { v.Field("value").Required() }), server.Handler(func(req *wcg.Request) response.Response { val := configs.Get(req, req.Param("key")) if val == nil { return response.NotFound(req) } else { val.Value = req.Form("value") configs.PutMulti(req, val) return response.NewJSONResponse(val) } })) }
func setupAPIGates(app *server.App) { var API = app.API() // -------------------------------------------------------------------------- // Gate API.GET("/gates/", server.Handler(func(req *wcg.Request) response.Response { return response.NewJSONResponse(gates.GetAll(req)) })) API.GET("/gates/:key.json", server.Handler(func(req *wcg.Request) response.Response { val := gates.Get(req, req.Param("key")) if val == nil { return response.NotFound(req) } else { return response.NewJSONResponse(val) } })) API.PUT("/gates/:key.json", middleware.ParseForm(nil), server.Handler(func(req *wcg.Request) response.Response { val := gates.Get(req, req.Param("key")) if val == nil { return response.NotFound(req) } else { val.UIDs = req.HTTPRequest().Form["uids"] gates.PutMulti(req, val) return response.NewJSONResponse(val) } })) }
func getIEPG(id string, f func(req *wcg.Request, iepg *pt.IEPG) response.Response) server.Handler { return server.Handler(func(req *wcg.Request) response.Response { _, one := IEPG.Get().Key(req.Param(id)).Cache(true).MustOne(req) if one == nil { return response.NotFound(req) } return f(req, one.(*pt.IEPG)) }) }
// NewInstance creates *Instance func NewInstance() *Instance { instance := &Instance{ Server: gae.NewServer(), } instance.Routes().Before(wcg.NewNamedHandler("init", func(res *wcg.Response, req *wcg.Request) { instance.once.Do(func() { req.Logger.Infof("----------- Server initialization ----------- ") if instance.Init != nil { instance.Init(req) } req.Logger.Infof("----------- Apps initialization ----------- ") for _, app := range instance.apps { if app.Init != nil { app.Init(req) } } req.Logger.Infof("----------- All init processes finished ----------- ") }) })) auth.Prepare(instance.Routes(), func() { instance.Routes().GET("/cron.yaml", wcg.NewNamedHandler("cron.yaml", func(res *wcg.Response, req *wcg.Request) { res.Header().Add("Content-Type", "text/yaml") res.WriteString("cron:\n") for _, app := range instance.apps { res.WriteString(app.Cron().ToYAML()) } })) instance.Routes().GET("/queue.yaml", wcg.NewNamedHandler("queue.yaml", func(res *wcg.Response, req *wcg.Request) { res.Header().Add("Content-Type", "text/yaml") res.WriteString("queue:\n") for _, app := range instance.apps { res.WriteString(app.Queue().ToYAML()) } })) }) instance.Routes().NotFound = Handler(func(req *wcg.Request) response.Response { return response.NotFound(req) }) return instance }
package messenger import ( "fmt" "lib/server" "lib/server/configs" "lib/server/response" "lib/wcg" ) // SetupHandler is a server.Handler to respond setup HTTP call from a Facebook server var SetupHandler = server.Handler(func(req *wcg.Request) response.Response { token := configs.GetValue(req, "facebook_messenger_verify_token") if token == "" { return response.NotFound(req) } if req.Query("hub.verify_token") != token { return response.BadRequest( req, fmt.Errorf( "token mismatch: hub.verify_token(%q) != facebook_messenger_verify_token(%q)", req.Query("hub.verify_token"), token, ), ) } return response.NewTextResponse(req.Query("hub.challenge")) }) // WebhookHandler is a server.Handler to respond setup HTTP call from a Facebook server var WebHookHandler = server.Handler(func(req *wcg.Request) response.Response { messages, perr := parseMessage(req.HTTPRequest().Body)
func setupAPIDatastore(app *server.App) { var API = app.API() if lib.IsOnLocalGAE() { API.GET("/datastore/cleanup/:kind.json", server.Handler(func(req *wcg.Request) response.Response { k := entities.FindKind(req.Param("kind"), req.Query("ns")) if k == nil { return response.NotFound(req) } num := k.DeleteQuery().MustCommit(req) return response.NewJSONResponse(map[string]int{ "entities": num, }) }), ) } API.GET("/datastore/export/:kind.json", middleware.APITokenOnly(), server.Handler(func(req *wcg.Request) response.Response { k := entities.FindKind(req.Param("kind"), req.Query("ns")) if k == nil { return response.NotFound(req) } // TODO: export with gzip return middleware.EntityStreaming( k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit( wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32), ), true, ).Handle(req) })) API.POST("/datastore/import/:kind.json", middleware.APITokenOnly(), server.Handler(func(req *wcg.Request) response.Response { k := entities.FindKind(req.Param("kind"), req.Query("ns")) if k == nil { return response.NotFound(req) } var line = 0 var total = 0 var errors = make([]*entityImportError, 0) var list = make([]interface{}, 0, 100) var keys = make([]*datastore.Key, 0, 100) var buff []byte var err error reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096) for err != io.EOF { buff, err = reader.ReadBytes('\n') line += 1 if err != nil && err != io.EOF { return response.InternalServerError(req, err) } buff = bytes.TrimSpace(buff) if len(buff) == 0 { // skip empty row continue } var row entityImportRow if err2 := json.Unmarshal(buff, &row); err2 != nil { errors = append(errors, &entityImportError{ Line: line, Error: fmt.Sprintf("%v - (row: %q)", err2, buff), }) continue } ent, err2 := k.NewEntityFromJSON([]byte(row.Entity)) if err2 != nil { } total = total + 1 keys = append(keys, buildKey(req, row.Key)) list = append(list, ent) if len(list) == 100 { k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list) list = make([]interface{}, 0, 100) keys = make([]*datastore.Key, 0, 100) } } if len(list) > 0 { k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list) } return response.NewJSONResponse(&entityImportResult{ Total: total, Lines: line - 1, Errors: errors, }) })) }
// Define endpoints to process an async task. func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig { if !strings.HasSuffix(path, "/") { panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path)) } api := helper.app.API() // /path/to/something/ => api-path-to-something p := api.Path(path) name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters name = strings.Replace(name, "/", "-", -1) // replace '/' with '-' config := &AsyncAPIConfig{ Queue: helper.app.Queue().DefinePush(name), app: helper.app, path: path, } // Endpoint that trigger the task. api.POST(path, Handler( func(req *wcg.Request) response.Response { for _, h := range config.triggerHandlers { resp := h.Handle(req) if resp != nil { req.Logger.Debugf("Async Task is not triggered.") return resp } } task := &models.AsyncAPITask{} task.ID = wcg.NewUUID() task.Path = path task.Query = req.HTTPRequest().URL.RawQuery task.Status = models.AsyncAPIStatusReady entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task) // Push a task taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode()) if err := config.Queue.PushTask(req, taskPath, nil); err != nil { return response.InternalServerError(req, err) } req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath) return response.NewJSONResponseWithStatus( AsyncTaskTriggerResponse{task.ID}, 201, ) }, )) // Endpoint that executes the task in background // This endpoint must be called by PushQueue api.POST(path+":taskid.json", Handler( func(req *wcg.Request) response.Response { if !request.IsTask(req) { req.Logger.Warnf( "Non system call to the async task endpoint: UserAgent=%s, IP=%s", req.HTTPRequest().UserAgent(), req.HTTPRequest().RemoteAddr, ) return response.APIOK } _, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req) if one == nil { req.Logger.Warnf( "Task %q not found: UserAgent=%s, IP=%s", req.Param("taskid"), req.HTTPRequest().UserAgent(), req.HTTPRequest().RemoteAddr, ) return response.APIOK } task := one.(*models.AsyncAPITask) logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name)) if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning { logger.Warnf("Task is not ready: Status=%s", task.Status) return response.APIOK } if task.Status == models.AsyncAPIStatusReady { task.StartAt = lib.Now() task.Status = models.AsyncAPIStatusRunning entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task) } var err error var progress *models.AsyncAPITaskProgress var resp response.Response func() { defer func() { if x := recover(); x != nil { logger.Errorf("Unhandle error recovered from the task: %v", x) err = fmt.Errorf("%v", x) } }() progress, err = config.processHandler(req, task) }() if progress != nil { task.Progress = append(task.Progress, *progress) } if progress != nil && progress.Next != nil { // update the entity (Progress field) entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task) // Push a next request into the queue // This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code) // The test environment uses the response JSON to call the next URL and that response is used only for this purpose. // If the code cannot push task for the next request, we should stop the task as failure. logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next) if !req.IsTest() { err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil) } if err == nil { return response.NewJSONResponse(progress.Next) } // if an error occurrs by PushTask, just stopping the task and update the datastore as failure. } // finished the task task.FinishAt = lib.Now() tt := task.FinishAt.Sub(task.StartAt) if err == nil { task.Status = models.AsyncAPIStatusSuccess logger.Infof("Task finished successfully (time: %s)", tt.String()) resp = response.APIOK } else { task.Error = err.Error() task.Status = models.AsyncAPIStatusFailure logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error) resp = response.APIInternalServerError } entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task) return resp }, )) // Endpoint that returns the task status. api.GET(path+":taskid.json", Handler( func(req *wcg.Request) response.Response { for _, h := range config.triggerHandlers { resp := h.Handle(req) if resp != nil { return resp } } _, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req) if one == nil { return response.NotFound(req) } task := one.(*models.AsyncAPITask) return response.NewJSONResponse( AsyncTaskMonitorResponse{ ID: task.ID, Status: task.Status, Progress: task.LastProgress(), }, ) }, )) return config }
func setupAPIOAuth2(app *server.App) { var API = app.API() API.GET("/oauth2/clients/", middleware.EntityAll(entities.OAuth2ClientSettings.Query())) API.GET("/oauth2/clients/:key.json", middleware.EntityGet(entities.OAuth2ClientSettings.Get().Cache(true), "key")) // Create an oauth2.Config object into session and redirect to the oauth2 endpoint. // /admin/oauth2/callback.html will finally authorize the callback code and store the oauth2.Config into datastore. API.POST("/oauth2/clients/", middleware.ParseForm(func(v *validators.FormValidator) { v.Field("key").Required() v.Field("client_id").Required() v.Field("client_secret").Required() v.Field("auth_url").Required() v.Field("token_url").Required() }), server.Handler(func(req *wcg.Request) response.Response { cfg := &oauth2.Config{ ClientID: req.Form("client_id"), ClientSecret: req.Form("client_secret"), Endpoint: oauth2.Endpoint{ AuthURL: req.Form("auth_url"), TokenURL: req.Form("token_url"), }, Scopes: strings.Split(req.Form("scopes"), ","), RedirectURL: wcg.AbsoluteURL( req, fmt.Sprintf("/api/admin/oauth2/callback/?oauth2_key=%s", req.Form("key")), ), } data := wcg.DataBag{} data.Set("key", req.Form("key")) data.Set("client_id", cfg.ClientID) data.Set("client_secret", cfg.ClientSecret) data.Set("auth_url", cfg.Endpoint.AuthURL) data.Set("token_url", cfg.Endpoint.TokenURL) data.Set("scopes", req.Form("scopes")) data.Set("redirect_url", cfg.RedirectURL) req.Session.SetData( fmt.Sprintf("admin.oauth2_%s", req.Form("key")), data, ) return response.NewRedirect(cfg.AuthCodeURL("state", oauth2.AccessTypeOffline), response.RedirectSeeOther) })) API.DELETE("/oauth2/clients/:client_id.json", middleware.EntityDelete(entities.OAuth2ClientSettings.Delete(), "client_id")) API.GET("/oauth2/callback/", server.Handler(func(req *wcg.Request) response.Response { key := req.Query("oauth2_key") code := req.Query("code") data, ok := req.Session.GetData(fmt.Sprintf("admin.oauth2_%s", key)) if !ok { return response.NotFound(req) } scopes, _ := data.Get("scopes") cfg := &oauth2.Config{} cfg.ClientID, _ = data.Get("client_id") cfg.ClientSecret, _ = data.Get("client_secret") cfg.Endpoint.AuthURL, _ = data.Get("auth_url") cfg.Endpoint.TokenURL, _ = data.Get("token_url") cfg.Scopes = strings.Split(scopes, ",") cfg.RedirectURL, _ = data.Get("redirect_url") token, err := cfg.Exchange(gae.NewContext(req), code) if err != nil { return response.BadRequest(req, err) } settings := models.NewOAuth2ClientSettings(cfg, token) settings.Key = key entities.OAuth2ClientSettings.Put().Key(key).MustUpdate(req, settings) return response.NewRedirect("/admin/oauth2/", response.RedirectSeeOther) })) }
func setupAPIStats(app *server.App) { var API = app.API() API.GET("/stats/servers/", middleware.EntityAll(Server.Query())) API.GET("/stats/nasdisks/", middleware.EntityAll(NASDisk.Query())) API.GET("/stats/servers/:server/:stats/", server.Handler( func(req *wcg.Request) response.Response { key, _ := Server.Get().Key(req.Param("server")).Cache(true).MustOne(req) if key == nil { req.Logger.Debugf("server %q is not found.", req.Param("server")) return response.NotFound(req) } var kind *entities.Kind switch req.Param("stats") { case "system": kind = SystemStats break case "cpu": kind = CPUStats break case "memory": kind = MemoryStats break case "disk": kind = DiskStats break case "filesystem": kind = FileSystemStats break case "network": kind = NetworkStats break default: req.Logger.Debugf("metric %q is invalid.", req.Param("stats")) return response.NotFound(req) } return execStatsQuery(req, kind.Query().Ancestor(key)) }, )) API.POST("/stats/metrics/", middleware.APITokenOnly(), server.Handler( func(req *wcg.Request) response.Response { ent, err := SystemMetric.CreateEntitiesFromJSON(req.HTTPRequest().Body) if err != nil { return response.BadRequest(req, err) } collection := home.NewStatsFromMetrics(ent.([]*home.SystemMetric)) if len(collection) == 0 { return response.BadRequest(req, fmt.Errorf("No stats are collected.")) } if collection[0].System.ServerName == "" { return response.BadRequest(req, fmt.Errorf("server_name is missing")) } for _, stats := range collection { key := getServerKey(req, stats.System.ServerName) id := fmt.Sprintf("%s.%d", key.StringID(), stats.Timestamp.Unix()) stats.System.ID = id stats.CPU.ID = id stats.Memory.ID = id for _, v := range stats.Disks { v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix()) } for _, v := range stats.Networks { v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix()) } SystemStats.Put().Parent(key).MustUpdate(req, stats.System) CPUStats.Put().Parent(key).MustUpdate(req, stats.CPU) MemoryStats.Put().Parent(key).MustUpdate(req, stats.Memory) DiskStats.PutMulti().Parent(key).MustUpdate(req, stats.Disks) NetworkStats.PutMulti().Parent(key).MustUpdate(req, stats.Networks) } return response.NewJSONResponse(true) }, )) API.POST("/stats/filesystem/", middleware.APITokenOnly(), server.Handler( func(req *wcg.Request) response.Response { ent, err := FileSystemStats.CreateEntitiesFromJSON(req.HTTPRequest().Body) if err != nil { return response.BadRequest(req, err) } filesystems := ent.([]*home.FileSystemStats) if filesystems[0].ServerName == "" { return response.BadRequest(req, fmt.Errorf("server_name is missing")) } key := getServerKey(req, filesystems[0].ServerName) for _, fs := range filesystems { fs.ID = fmt.Sprintf("%s.%s.%d", fs.ServerName, fs.Name, fs.Timestamp.Unix()) } FileSystemStats.PutMulti().Parent(key).MustUpdate(req, filesystems) return response.NewJSONResponse(true) }, )) API.POST("/stats/smart/", middleware.APITokenOnly(), server.Handler( func(req *wcg.Request) response.Response { ent, err := SMARTStats.CreateEntitiesFromJSON(req.HTTPRequest().Body) if err != nil { return response.BadRequest(req, err) } smartstats := ent.([]*home.SMARTStats) if smartstats[0].Serial == "" { return response.BadRequest(req, fmt.Errorf("serial is missing")) } key := getNASDiskKey(req, smartstats[0].Serial) for _, smart := range smartstats { smart.ID = fmt.Sprintf("%s.%d", smart.Serial, smart.Timestamp.Unix()) } SMARTStats.PutMulti().Parent(key).MustUpdate(req, smartstats) return response.NewJSONResponse(true) }, )) }