// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
name := conf.RequiredString("program")
backendURL := conf.OptionalString("backendURL", "")
appConfig := conf.OptionalObject("appConfig")
// TODO(mpl): add an auth token in the extra config of the dev server config,
// that the hello app can use to setup a status handler than only responds
// to requests with that token.
if err := conf.Validate(); err != nil {
return nil, err
}
if apiHost == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
}
if appHandlerPrefix == "" {
return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
}
if backendURL == "" {
var err error
// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
// now (instead of letting the app itself do it), because we need to know it in advance in order
// to set the app handler's proxy.
backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
if err != nil {
return nil, err
}
}
username, password := auth.RandToken(20), auth.RandToken(20)
camliAuth := username + ":" + password
basicAuth := auth.NewBasicAuth(username, password)
envVars := map[string]string{
"CAMLI_API_HOST": apiHost,
"CAMLI_AUTH": camliAuth,
"CAMLI_APP_BACKEND_URL": backendURL,
}
if appConfig != nil {
envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
}
proxyURL, err := url.Parse(backendURL)
if err != nil {
return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
}
return &Handler{
name: name,
envVars: envVars,
auth: basicAuth,
appConfig: appConfig,
proxy: httputil.NewSingleHostReverseProxy(proxyURL),
backendURL: backendURL,
}, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
path = config.RequiredString("path")
maxFileSize = config.OptionalInt("maxFileSize", 0)
indexConf = config.OptionalObject("metaIndex")
)
if err := config.Validate(); err != nil {
return nil, err
}
return newStorage(path, int64(maxFileSize), indexConf)
}
// FromJSONConfig creates an HandlerConfig from the contents of config.
// serverBaseURL is used if it is not found in config.
func FromJSONConfig(config jsonconfig.Obj, serverBaseURL string) (HandlerConfig, error) {
hc := HandlerConfig{
Program: config.RequiredString("program"),
Prefix: config.RequiredString("prefix"),
BackendURL: config.OptionalString("backendURL", ""),
Listen: config.OptionalString("listen", ""),
APIHost: config.OptionalString("apiHost", ""),
ServerListen: config.OptionalString("serverListen", ""),
ServerBaseURL: config.OptionalString("serverBaseURL", ""),
AppConfig: config.OptionalObject("appConfig"),
}
if hc.ServerBaseURL == "" {
hc.ServerBaseURL = serverBaseURL
}
if err := config.Validate(); err != nil {
return HandlerConfig{}, err
}
return hc, nil
}
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) {
hc := HostConfig{
BaseURL: ld.BaseURL(),
Prefix: ld.MyPrefix(),
}
ClientId := make(map[string]string)
ClientSecret := make(map[string]string)
for k, _ := range importers {
var clientId, clientSecret string
if impConf := cfg.OptionalObject(k); impConf != nil {
clientId = impConf.OptionalString("clientID", "")
clientSecret = impConf.OptionalString("clientSecret", "")
// Special case: allow clientSecret to be of form "clientId:clientSecret"
// if the clientId is empty.
if clientId == "" && strings.Contains(clientSecret, ":") {
if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 {
clientId, clientSecret = f[0], f[1]
}
}
if err := impConf.Validate(); err != nil {
return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err)
}
ClientId[k] = clientId
ClientSecret[k] = clientSecret
}
}
if err := cfg.Validate(); err != nil {
return nil, err
}
hc.ClientId = ClientId
hc.ClientSecret = ClientSecret
host, err := NewHost(hc)
if err != nil {
return nil, err
}
host.didInit.Add(1)
return host, nil
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
from = conf.RequiredString("from")
to = conf.RequiredString("to")
fullSync = conf.OptionalBool("fullSyncOnStart", false)
blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false)
idle = conf.OptionalBool("idle", false)
queueConf = conf.OptionalObject("queue")
copierPoolSize = conf.OptionalInt("copierPoolSize", 5)
validate = conf.OptionalBool("validateOnStart", validateOnStartDefault)
)
if err := conf.Validate(); err != nil {
return nil, err
}
if idle {
return newIdleSyncHandler(from, to), nil
}
if len(queueConf) == 0 {
return nil, errors.New(`Missing required "queue" object`)
}
q, err := sorted.NewKeyValue(queueConf)
if err != nil {
return nil, err
}
isToIndex := false
fromBs, err := ld.GetStorage(from)
if err != nil {
return nil, err
}
toBs, err := ld.GetStorage(to)
if err != nil {
return nil, err
}
if _, ok := fromBs.(*index.Index); !ok {
if _, ok := toBs.(*index.Index); ok {
isToIndex = true
}
}
sh := newSyncHandler(from, to, fromBs, toBs, q)
sh.toIndex = isToIndex
sh.copierPoolSize = copierPoolSize
if err := sh.readQueueToMemory(); err != nil {
return nil, fmt.Errorf("Error reading sync queue to memory: %v", err)
}
if fullSync || blockFullSync {
sh.logf("Doing full sync")
didFullSync := make(chan bool, 1)
go func() {
for {
n := sh.runSync("pending blobs queue", sh.enumeratePendingBlobs)
if n > 0 {
sh.logf("Queue sync copied %d blobs", n)
continue
}
break
}
n := sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs))
sh.logf("Full sync copied %d blobs", n)
didFullSync <- true
sh.syncLoop()
}()
if blockFullSync {
sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
sh.logf("Full sync complete.")
}
} else {
go sh.syncLoop()
}
if validate {
go sh.startFullValidation()
}
blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue)
return sh, nil
}
// GenerateClientConfig retuns a client configuration which can be used to
// access a server defined by the provided low-level server configuration.
func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) {
missingConfig := func(param string) (*Config, error) {
return nil, fmt.Errorf("required value for %q not found", param)
}
if serverConfig == nil {
return nil, errors.New("server config is a required parameter")
}
param := "auth"
auth := serverConfig.OptionalString(param, "")
if auth == "" {
return missingConfig(param)
}
listen := serverConfig.OptionalString("listen", "")
baseURL := serverConfig.OptionalString("baseURL", "")
if listen == "" {
listen = baseURL
}
if listen == "" {
return nil, errors.New("required value for 'listen' or 'baseURL' not found")
}
https := serverConfig.OptionalBool("https", false)
if !strings.HasPrefix(listen, "http://") && !strings.HasPrefix(listen, "https://") {
if !https {
listen = "http://" + listen
} else {
listen = "https://" + listen
}
}
httpsCert := serverConfig.OptionalString("httpsCert", "")
// TODO(mpl): See if we can detect that the cert is not self-signed,and in
// that case not add it to the trustedCerts
var trustedList []string
if https && httpsCert != "" {
certPEMBlock, err := wkfs.ReadFile(httpsCert)
if err != nil {
return nil, fmt.Errorf("could not read certificate: %v", err)
}
sig, err := httputil.CertFingerprint(certPEMBlock)
if err != nil {
return nil, fmt.Errorf("could not get fingerprints of certificate: %v", err)
}
trustedList = []string{sig}
}
param = "prefixes"
prefixes := serverConfig.OptionalObject(param)
if len(prefixes) == 0 {
return missingConfig(param)
}
param = "/sighelper/"
sighelper := prefixes.OptionalObject(param)
if len(sighelper) == 0 {
return missingConfig(param)
}
param = "handlerArgs"
handlerArgs := sighelper.OptionalObject(param)
if len(handlerArgs) == 0 {
return missingConfig(param)
}
param = "keyId"
keyId := handlerArgs.OptionalString(param, "")
if keyId == "" {
return missingConfig(param)
}
param = "secretRing"
secretRing := handlerArgs.OptionalString(param, "")
if secretRing == "" {
return missingConfig(param)
}
return &Config{
Servers: map[string]*Server{
"default": {
Server: listen,
Auth: auth,
IsDefault: true,
TrustedCerts: trustedList,
},
},
Identity: keyId,
IdentitySecretRing: secretRing,
IgnoredFiles: []string{".DS_Store", "*~"},
}, nil
}
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
ui := &UIHandler{
prefix: ld.MyPrefix(),
sourceRoot: conf.OptionalString("sourceRoot", ""),
resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes",
constants.DefaultMaxResizeMem))),
}
cachePrefix := conf.OptionalString("cache", "")
scaledImageConf := conf.OptionalObject("scaledImage")
if err = conf.Validate(); err != nil {
return
}
scaledImageKV, err := newKVOrNil(scaledImageConf)
if err != nil {
return nil, fmt.Errorf("in UI handler's scaledImage: %v", err)
}
if scaledImageKV != nil && cachePrefix == "" {
return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache")
}
if cachePrefix != "" {
bs, err := ld.GetStorage(cachePrefix)
if err != nil {
return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
}
ui.Cache = bs
ui.thumbMeta = NewThumbMeta(scaledImageKV)
}
if ui.sourceRoot == "" {
ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT")
if uistatic.IsAppEngine {
if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot,
filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil {
hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+
" and the default sourceRoot dir %v does not exist or does not contain"+
" \"server/camlistored/ui/index.html\". devcam appengine can do that for you.",
uistatic.GaeSourceRoot)
log.Print(hint)
return nil, errors.New("No sourceRoot found; UI not available.")
}
log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot)
ui.sourceRoot = uistatic.GaeSourceRoot
}
if ui.sourceRoot == "" && uistatic.Files.IsEmpty() {
ui.sourceRoot, err = osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Printf("Warning: server not compiled with linked-in UI resources (HTML, JS, CSS), and camlistore.org not found in GOPATH.")
} else {
log.Printf("Using UI resources (HTML, JS, CSS) from disk, under %v", ui.sourceRoot)
}
}
}
if ui.sourceRoot != "" {
ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui"))
// Ignore any fileembed files:
Files = &fileembed.Files{
DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")),
}
uistatic.Files = &fileembed.Files{
DirFallback: ui.uiDir,
Listable: true,
// In dev_appserver, allow edit-and-reload without
// restarting. In production, though, it's faster to just
// slurp it in.
SlurpToMemory: uistatic.IsProdAppEngine,
}
}
ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot)
if err != nil {
return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err)
}
if ui.sourceRoot != "" {
ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "react"), "react.js")
if err != nil {
return nil, fmt.Errorf("Could not make react handler: %s", err)
}
ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "glitch"), "npc_piggy__x1_walk_png_1354829432.png")
if err != nil {
return nil, fmt.Errorf("Could not make glitch handler: %s", err)
}
ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "fontawesome"), "css/font-awesome.css")
if err != nil {
return nil, fmt.Errorf("Could not make fontawesome handler: %s", err)
}
ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "less"), "less.js")
if err != nil {
return nil, fmt.Errorf("Could not make less handler: %s", err)
}
}
rootPrefix, _, err := ld.FindHandlerByType("root")
if err != nil {
return nil, errors.New("No root handler configured, which is necessary for the ui handler")
}
if h, err := ld.GetHandler(rootPrefix); err == nil {
ui.root = h.(*RootHandler)
ui.root.registerUIHandler(ui)
} else {
return nil, errors.New("failed to find the 'root' handler")
}
return ui, nil
}