Пример #1
0
func setupAPIConfigs(app *server.App) {
	var API = app.API()
	// --------------------------------------------------------------------------
	// Sever Config
	API.GET("/configs/",
		server.Handler(func(req *wcg.Request) response.Response {
			return response.NewJSONResponse(configs.GetAll(req))
		}))

	API.GET("/configs/:key.json",
		server.Handler(func(req *wcg.Request) response.Response {
			val := configs.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				return response.NewJSONResponse(val)
			}
		}),
	)

	API.PUT("/configs/:key.json",
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("value").Required()
		}),
		server.Handler(func(req *wcg.Request) response.Response {
			val := configs.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				val.Value = req.Form("value")
				configs.PutMulti(req, val)
				return response.NewJSONResponse(val)
			}
		}))
}
Пример #2
0
func setupAPIGates(app *server.App) {
	var API = app.API()

	// --------------------------------------------------------------------------
	// Gate
	API.GET("/gates/",
		server.Handler(func(req *wcg.Request) response.Response {
			return response.NewJSONResponse(gates.GetAll(req))
		}))

	API.GET("/gates/:key.json",
		server.Handler(func(req *wcg.Request) response.Response {
			val := gates.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				return response.NewJSONResponse(val)
			}
		}))

	API.PUT("/gates/:key.json",
		middleware.ParseForm(nil),
		server.Handler(func(req *wcg.Request) response.Response {
			val := gates.Get(req, req.Param("key"))
			if val == nil {
				return response.NotFound(req)
			} else {
				val.UIDs = req.HTTPRequest().Form["uids"]
				gates.PutMulti(req, val)
				return response.NewJSONResponse(val)
			}
		}))
}
Пример #3
0
func getIEPG(id string, f func(req *wcg.Request, iepg *pt.IEPG) response.Response) server.Handler {
	return server.Handler(func(req *wcg.Request) response.Response {
		_, one := IEPG.Get().Key(req.Param(id)).Cache(true).MustOne(req)
		if one == nil {
			return response.NotFound(req)
		}
		return f(req, one.(*pt.IEPG))
	})
}
Пример #4
0
// NewInstance creates *Instance
func NewInstance() *Instance {
	instance := &Instance{
		Server: gae.NewServer(),
	}
	instance.Routes().Before(wcg.NewNamedHandler("init", func(res *wcg.Response, req *wcg.Request) {
		instance.once.Do(func() {
			req.Logger.Infof("----------- Server initialization ----------- ")
			if instance.Init != nil {
				instance.Init(req)
			}
			req.Logger.Infof("----------- Apps initialization ----------- ")
			for _, app := range instance.apps {
				if app.Init != nil {
					app.Init(req)
				}
			}
			req.Logger.Infof("----------- All init processes finished ----------- ")
		})
	}))
	auth.Prepare(instance.Routes(), func() {
		instance.Routes().GET("/cron.yaml", wcg.NewNamedHandler("cron.yaml", func(res *wcg.Response, req *wcg.Request) {
			res.Header().Add("Content-Type", "text/yaml")
			res.WriteString("cron:\n")
			for _, app := range instance.apps {
				res.WriteString(app.Cron().ToYAML())
			}
		}))
		instance.Routes().GET("/queue.yaml", wcg.NewNamedHandler("queue.yaml", func(res *wcg.Response, req *wcg.Request) {
			res.Header().Add("Content-Type", "text/yaml")
			res.WriteString("queue:\n")
			for _, app := range instance.apps {
				res.WriteString(app.Queue().ToYAML())
			}
		}))
	})

	instance.Routes().NotFound = Handler(func(req *wcg.Request) response.Response {
		return response.NotFound(req)
	})
	return instance
}
Пример #5
0
package messenger

import (
	"fmt"
	"lib/server"
	"lib/server/configs"
	"lib/server/response"
	"lib/wcg"
)

// SetupHandler is a server.Handler to respond setup HTTP call from a Facebook server
var SetupHandler = server.Handler(func(req *wcg.Request) response.Response {
	token := configs.GetValue(req, "facebook_messenger_verify_token")
	if token == "" {
		return response.NotFound(req)
	}
	if req.Query("hub.verify_token") != token {
		return response.BadRequest(
			req,
			fmt.Errorf(
				"token mismatch: hub.verify_token(%q) != facebook_messenger_verify_token(%q)",
				req.Query("hub.verify_token"), token,
			),
		)
	}
	return response.NewTextResponse(req.Query("hub.challenge"))
})

// WebhookHandler is a server.Handler to respond setup HTTP call from a Facebook server
var WebHookHandler = server.Handler(func(req *wcg.Request) response.Response {
	messages, perr := parseMessage(req.HTTPRequest().Body)
Пример #6
0
func setupAPIDatastore(app *server.App) {
	var API = app.API()

	if lib.IsOnLocalGAE() {
		API.GET("/datastore/cleanup/:kind.json",
			server.Handler(func(req *wcg.Request) response.Response {
				k := entities.FindKind(req.Param("kind"), req.Query("ns"))
				if k == nil {
					return response.NotFound(req)
				}
				num := k.DeleteQuery().MustCommit(req)
				return response.NewJSONResponse(map[string]int{
					"entities": num,
				})
			}),
		)
	}

	API.GET("/datastore/export/:kind.json",
		middleware.APITokenOnly(),
		server.Handler(func(req *wcg.Request) response.Response {
			k := entities.FindKind(req.Param("kind"), req.Query("ns"))
			if k == nil {
				return response.NotFound(req)
			}
			// TODO: export with gzip
			return middleware.EntityStreaming(
				k.Query().Order(fmt.Sprintf("-%s", k.TimestampFieldName)).Limit(
					wcg.ParseInt(req.Query("limit"), 100, -1, math.MaxInt32),
				),
				true,
			).Handle(req)
		}))

	API.POST("/datastore/import/:kind.json",
		middleware.APITokenOnly(),
		server.Handler(func(req *wcg.Request) response.Response {
			k := entities.FindKind(req.Param("kind"), req.Query("ns"))
			if k == nil {
				return response.NotFound(req)
			}
			var line = 0
			var total = 0
			var errors = make([]*entityImportError, 0)
			var list = make([]interface{}, 0, 100)
			var keys = make([]*datastore.Key, 0, 100)
			var buff []byte
			var err error
			reader := bufio.NewReaderSize(req.HTTPRequest().Body, 4096)
			for err != io.EOF {
				buff, err = reader.ReadBytes('\n')
				line += 1
				if err != nil && err != io.EOF {
					return response.InternalServerError(req, err)
				}
				buff = bytes.TrimSpace(buff)
				if len(buff) == 0 { // skip empty row
					continue
				}
				var row entityImportRow
				if err2 := json.Unmarshal(buff, &row); err2 != nil {
					errors = append(errors, &entityImportError{
						Line:  line,
						Error: fmt.Sprintf("%v - (row: %q)", err2, buff),
					})
					continue
				}
				ent, err2 := k.NewEntityFromJSON([]byte(row.Entity))
				if err2 != nil {
				}
				total = total + 1
				keys = append(keys, buildKey(req, row.Key))
				list = append(list, ent)
				if len(list) == 100 {
					k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
					list = make([]interface{}, 0, 100)
					keys = make([]*datastore.Key, 0, 100)
				}
			}
			if len(list) > 0 {
				k.PutMulti().DatastoreKeys(keys...).DoNotUpdateTimestamp(true).MustUpdate(req, list)
			}
			return response.NewJSONResponse(&entityImportResult{
				Total:  total,
				Lines:  line - 1,
				Errors: errors,
			})
		}))
}
Пример #7
0
// Define endpoints to process an async task.
func (helper *AsyncAPIHelper) Define(path string) *AsyncAPIConfig {
	if !strings.HasSuffix(path, "/") {
		panic(fmt.Errorf("Task endpoint must end with '/' (%s)", path))
	}
	api := helper.app.API()
	// /path/to/something/ => api-path-to-something
	p := api.Path(path)
	name := strings.Replace(p[1:len(p)-1], ":", "", -1) // remove parameters
	name = strings.Replace(name, "/", "-", -1)          // replace '/' with '-'
	config := &AsyncAPIConfig{
		Queue: helper.app.Queue().DefinePush(name),
		app:   helper.app,
		path:  path,
	}

	// Endpoint that trigger the task.
	api.POST(path, Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					req.Logger.Debugf("Async Task is not triggered.")
					return resp
				}
			}
			task := &models.AsyncAPITask{}
			task.ID = wcg.NewUUID()
			task.Path = path
			task.Query = req.HTTPRequest().URL.RawQuery
			task.Status = models.AsyncAPIStatusReady
			entities.AsyncAPITask.Put().Key(string(task.ID)).MustUpdate(req, task)
			// Push a task
			taskPath := fmt.Sprintf("%s%s.json?%s", req.URL().Path, string(task.ID), req.HTTPRequest().URL.Query().Encode())
			if err := config.Queue.PushTask(req, taskPath, nil); err != nil {
				return response.InternalServerError(req, err)
			}
			req.Logger.Infof("Triggered an async task [id:%s path:%s]", task.ID, taskPath)
			return response.NewJSONResponseWithStatus(
				AsyncTaskTriggerResponse{task.ID},
				201,
			)
		},
	))

	// Endpoint that executes the task in background
	// This endpoint must be called by PushQueue
	api.POST(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			if !request.IsTask(req) {
				req.Logger.Warnf(
					"Non system call to the async task endpoint: UserAgent=%s, IP=%s",
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}

			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				req.Logger.Warnf(
					"Task %q not found: UserAgent=%s, IP=%s",
					req.Param("taskid"),
					req.HTTPRequest().UserAgent(),
					req.HTTPRequest().RemoteAddr,
				)
				return response.APIOK
			}
			task := one.(*models.AsyncAPITask)
			logger := wcg.NewLoggerWithPrefix(req, fmt.Sprintf("[AsyncTask: %s (queue: %s)]", task.ID, name))
			if task.Status != models.AsyncAPIStatusReady && task.Status != models.AsyncAPIStatusRunning {
				logger.Warnf("Task is not ready: Status=%s", task.Status)
				return response.APIOK
			}

			if task.Status == models.AsyncAPIStatusReady {
				task.StartAt = lib.Now()
				task.Status = models.AsyncAPIStatusRunning
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			}

			var err error
			var progress *models.AsyncAPITaskProgress
			var resp response.Response
			func() {
				defer func() {
					if x := recover(); x != nil {
						logger.Errorf("Unhandle error recovered from the task: %v", x)
						err = fmt.Errorf("%v", x)
					}
				}()
				progress, err = config.processHandler(req, task)
			}()

			if progress != nil {
				task.Progress = append(task.Progress, *progress)
			}
			if progress != nil && progress.Next != nil {
				// update the entity (Progress field)
				entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)

				// Push a next request into the queue
				// This is not required in test environment as test code manually call the next request (see testhelper.AsyncTaskTestRunner#Run code)
				// The test environment uses the response JSON to call the next URL and that response is used only for this purpose.
				// If the code cannot push task for the next request, we should stop the task as failure.
				logger.Infof("Task needs to call recursively: Next parameter is %v", progress.Next)
				if !req.IsTest() {
					err = config.Queue.PushTask(req, fmt.Sprintf("%s?%s", req.URL().Path, progress.Next.Encode()), nil)
				}
				if err == nil {
					return response.NewJSONResponse(progress.Next)
				}
				// if an error occurrs by PushTask, just stopping the task and update the datastore as failure.
			}
			// finished the task
			task.FinishAt = lib.Now()
			tt := task.FinishAt.Sub(task.StartAt)
			if err == nil {
				task.Status = models.AsyncAPIStatusSuccess
				logger.Infof("Task finished successfully (time: %s)", tt.String())
				resp = response.APIOK
			} else {
				task.Error = err.Error()
				task.Status = models.AsyncAPIStatusFailure
				logger.Errorf("Task failed (time: %s): %s", tt.String(), task.Error)
				resp = response.APIInternalServerError
			}
			entities.AsyncAPITask.Put().Key(req.Param("taskid")).MustUpdate(req, task)
			return resp
		},
	))

	// Endpoint that returns the task status.
	api.GET(path+":taskid.json", Handler(
		func(req *wcg.Request) response.Response {
			for _, h := range config.triggerHandlers {
				resp := h.Handle(req)
				if resp != nil {
					return resp
				}
			}
			_, one := entities.AsyncAPITask.Get().Key(req.Param("taskid")).MustOne(req)
			if one == nil {
				return response.NotFound(req)
			}
			task := one.(*models.AsyncAPITask)
			return response.NewJSONResponse(
				AsyncTaskMonitorResponse{
					ID:       task.ID,
					Status:   task.Status,
					Progress: task.LastProgress(),
				},
			)
		},
	))

	return config
}
Пример #8
0
func setupAPIOAuth2(app *server.App) {
	var API = app.API()

	API.GET("/oauth2/clients/",
		middleware.EntityAll(entities.OAuth2ClientSettings.Query()))

	API.GET("/oauth2/clients/:key.json",
		middleware.EntityGet(entities.OAuth2ClientSettings.Get().Cache(true), "key"))

	// Create an oauth2.Config object into session and redirect to the oauth2 endpoint.
	// /admin/oauth2/callback.html will finally authorize the callback code and store the oauth2.Config into datastore.
	API.POST("/oauth2/clients/",
		middleware.ParseForm(func(v *validators.FormValidator) {
			v.Field("key").Required()
			v.Field("client_id").Required()
			v.Field("client_secret").Required()
			v.Field("auth_url").Required()
			v.Field("token_url").Required()
		}),
		server.Handler(func(req *wcg.Request) response.Response {
			cfg := &oauth2.Config{
				ClientID:     req.Form("client_id"),
				ClientSecret: req.Form("client_secret"),
				Endpoint: oauth2.Endpoint{
					AuthURL:  req.Form("auth_url"),
					TokenURL: req.Form("token_url"),
				},
				Scopes: strings.Split(req.Form("scopes"), ","),
				RedirectURL: wcg.AbsoluteURL(
					req,
					fmt.Sprintf("/api/admin/oauth2/callback/?oauth2_key=%s", req.Form("key")),
				),
			}
			data := wcg.DataBag{}
			data.Set("key", req.Form("key"))
			data.Set("client_id", cfg.ClientID)
			data.Set("client_secret", cfg.ClientSecret)
			data.Set("auth_url", cfg.Endpoint.AuthURL)
			data.Set("token_url", cfg.Endpoint.TokenURL)
			data.Set("scopes", req.Form("scopes"))
			data.Set("redirect_url", cfg.RedirectURL)
			req.Session.SetData(
				fmt.Sprintf("admin.oauth2_%s", req.Form("key")),
				data,
			)
			return response.NewRedirect(cfg.AuthCodeURL("state", oauth2.AccessTypeOffline), response.RedirectSeeOther)
		}))

	API.DELETE("/oauth2/clients/:client_id.json",
		middleware.EntityDelete(entities.OAuth2ClientSettings.Delete(), "client_id"))

	API.GET("/oauth2/callback/",
		server.Handler(func(req *wcg.Request) response.Response {
			key := req.Query("oauth2_key")
			code := req.Query("code")
			data, ok := req.Session.GetData(fmt.Sprintf("admin.oauth2_%s", key))
			if !ok {
				return response.NotFound(req)
			}
			scopes, _ := data.Get("scopes")
			cfg := &oauth2.Config{}
			cfg.ClientID, _ = data.Get("client_id")
			cfg.ClientSecret, _ = data.Get("client_secret")
			cfg.Endpoint.AuthURL, _ = data.Get("auth_url")
			cfg.Endpoint.TokenURL, _ = data.Get("token_url")
			cfg.Scopes = strings.Split(scopes, ",")
			cfg.RedirectURL, _ = data.Get("redirect_url")
			token, err := cfg.Exchange(gae.NewContext(req), code)
			if err != nil {

				return response.BadRequest(req, err)
			}
			settings := models.NewOAuth2ClientSettings(cfg, token)
			settings.Key = key
			entities.OAuth2ClientSettings.Put().Key(key).MustUpdate(req, settings)
			return response.NewRedirect("/admin/oauth2/", response.RedirectSeeOther)
		}))

}
Пример #9
0
func setupAPIStats(app *server.App) {
	var API = app.API()

	API.GET("/stats/servers/",
		middleware.EntityAll(Server.Query()))

	API.GET("/stats/nasdisks/",
		middleware.EntityAll(NASDisk.Query()))

	API.GET("/stats/servers/:server/:stats/",
		server.Handler(
			func(req *wcg.Request) response.Response {
				key, _ := Server.Get().Key(req.Param("server")).Cache(true).MustOne(req)
				if key == nil {
					req.Logger.Debugf("server %q is not found.", req.Param("server"))
					return response.NotFound(req)
				}
				var kind *entities.Kind
				switch req.Param("stats") {
				case "system":
					kind = SystemStats
					break
				case "cpu":
					kind = CPUStats
					break
				case "memory":
					kind = MemoryStats
					break
				case "disk":
					kind = DiskStats
					break
				case "filesystem":
					kind = FileSystemStats
					break
				case "network":
					kind = NetworkStats
					break
				default:
					req.Logger.Debugf("metric %q is invalid.", req.Param("stats"))
					return response.NotFound(req)
				}
				return execStatsQuery(req, kind.Query().Ancestor(key))
			},
		))

	API.POST("/stats/metrics/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := SystemMetric.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				collection := home.NewStatsFromMetrics(ent.([]*home.SystemMetric))
				if len(collection) == 0 {
					return response.BadRequest(req, fmt.Errorf("No stats are collected."))
				}
				if collection[0].System.ServerName == "" {
					return response.BadRequest(req, fmt.Errorf("server_name is missing"))
				}
				for _, stats := range collection {
					key := getServerKey(req, stats.System.ServerName)
					id := fmt.Sprintf("%s.%d", key.StringID(), stats.Timestamp.Unix())
					stats.System.ID = id
					stats.CPU.ID = id
					stats.Memory.ID = id
					for _, v := range stats.Disks {
						v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
					}
					for _, v := range stats.Networks {
						v.ID = fmt.Sprintf("%s.%s.%d", key.StringID(), v.DeviceName, stats.Timestamp.Unix())
					}
					SystemStats.Put().Parent(key).MustUpdate(req, stats.System)
					CPUStats.Put().Parent(key).MustUpdate(req, stats.CPU)
					MemoryStats.Put().Parent(key).MustUpdate(req, stats.Memory)
					DiskStats.PutMulti().Parent(key).MustUpdate(req, stats.Disks)
					NetworkStats.PutMulti().Parent(key).MustUpdate(req, stats.Networks)
				}
				return response.NewJSONResponse(true)
			},
		))

	API.POST("/stats/filesystem/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := FileSystemStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				filesystems := ent.([]*home.FileSystemStats)
				if filesystems[0].ServerName == "" {
					return response.BadRequest(req, fmt.Errorf("server_name is missing"))
				}
				key := getServerKey(req, filesystems[0].ServerName)
				for _, fs := range filesystems {
					fs.ID = fmt.Sprintf("%s.%s.%d", fs.ServerName, fs.Name, fs.Timestamp.Unix())
				}
				FileSystemStats.PutMulti().Parent(key).MustUpdate(req, filesystems)
				return response.NewJSONResponse(true)
			},
		))

	API.POST("/stats/smart/",
		middleware.APITokenOnly(),
		server.Handler(
			func(req *wcg.Request) response.Response {
				ent, err := SMARTStats.CreateEntitiesFromJSON(req.HTTPRequest().Body)
				if err != nil {
					return response.BadRequest(req, err)
				}
				smartstats := ent.([]*home.SMARTStats)
				if smartstats[0].Serial == "" {
					return response.BadRequest(req, fmt.Errorf("serial is missing"))
				}
				key := getNASDiskKey(req, smartstats[0].Serial)
				for _, smart := range smartstats {
					smart.ID = fmt.Sprintf("%s.%d", smart.Serial, smart.Timestamp.Unix())
				}
				SMARTStats.PutMulti().Parent(key).MustUpdate(req, smartstats)
				return response.NewJSONResponse(true)
			},
		))
}