// NewHandler returns a Handler that proxies requests to an app. Start() on the // Handler starts the app. // The apiHost must end in a slash and is the camlistored API server for the app // process to hit. // The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted. // It must end in a slash, and be at minimum "/". // The conf object has the following members, related to the vars described in // doc/app-environment.txt: // "program", string, required. File name of the app's program executable. Either // an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH. // "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL. // "appConfig", object, optional. Additional configuration that the app can request from Camlistore. func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) { // TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available. name := conf.RequiredString("program") backendURL := conf.OptionalString("backendURL", "") appConfig := conf.OptionalObject("appConfig") // TODO(mpl): add an auth token in the extra config of the dev server config, // that the hello app can use to setup a status handler than only responds // to requests with that token. if err := conf.Validate(); err != nil { return nil, err } if apiHost == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name) } if appHandlerPrefix == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name) } if backendURL == "" { var err error // If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL // now (instead of letting the app itself do it), because we need to know it in advance in order // to set the app handler's proxy. backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix) if err != nil { return nil, err } } username, password := auth.RandToken(20), auth.RandToken(20) camliAuth := username + ":" + password basicAuth := auth.NewBasicAuth(username, password) envVars := map[string]string{ "CAMLI_API_HOST": apiHost, "CAMLI_AUTH": camliAuth, "CAMLI_APP_BACKEND_URL": backendURL, } if appConfig != nil { envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json" } proxyURL, err := url.Parse(backendURL) if err != nil { return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err) } return &Handler{ name: name, envVars: envVars, auth: basicAuth, appConfig: appConfig, proxy: httputil.NewSingleHostReverseProxy(proxyURL), backendURL: backendURL, }, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { var ( path = config.RequiredString("path") maxFileSize = config.OptionalInt("maxFileSize", 0) indexConf = config.OptionalObject("metaIndex") ) if err := config.Validate(); err != nil { return nil, err } return newStorage(path, int64(maxFileSize), indexConf) }
// FromJSONConfig creates an HandlerConfig from the contents of config. // serverBaseURL is used if it is not found in config. func FromJSONConfig(config jsonconfig.Obj, serverBaseURL string) (HandlerConfig, error) { hc := HandlerConfig{ Program: config.RequiredString("program"), Prefix: config.RequiredString("prefix"), BackendURL: config.OptionalString("backendURL", ""), Listen: config.OptionalString("listen", ""), APIHost: config.OptionalString("apiHost", ""), ServerListen: config.OptionalString("serverListen", ""), ServerBaseURL: config.OptionalString("serverBaseURL", ""), AppConfig: config.OptionalObject("appConfig"), } if hc.ServerBaseURL == "" { hc.ServerBaseURL = serverBaseURL } if err := config.Validate(); err != nil { return HandlerConfig{}, err } return hc, nil }
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) { hc := HostConfig{ BaseURL: ld.BaseURL(), Prefix: ld.MyPrefix(), } ClientId := make(map[string]string) ClientSecret := make(map[string]string) for k, _ := range importers { var clientId, clientSecret string if impConf := cfg.OptionalObject(k); impConf != nil { clientId = impConf.OptionalString("clientID", "") clientSecret = impConf.OptionalString("clientSecret", "") // Special case: allow clientSecret to be of form "clientId:clientSecret" // if the clientId is empty. if clientId == "" && strings.Contains(clientSecret, ":") { if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 { clientId, clientSecret = f[0], f[1] } } if err := impConf.Validate(); err != nil { return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err) } ClientId[k] = clientId ClientSecret[k] = clientSecret } } if err := cfg.Validate(); err != nil { return nil, err } hc.ClientId = ClientId hc.ClientSecret = ClientSecret host, err := NewHost(hc) if err != nil { return nil, err } host.didInit.Add(1) return host, nil }
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { var ( from = conf.RequiredString("from") to = conf.RequiredString("to") fullSync = conf.OptionalBool("fullSyncOnStart", false) blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false) idle = conf.OptionalBool("idle", false) queueConf = conf.OptionalObject("queue") copierPoolSize = conf.OptionalInt("copierPoolSize", 5) validate = conf.OptionalBool("validateOnStart", validateOnStartDefault) ) if err := conf.Validate(); err != nil { return nil, err } if idle { return newIdleSyncHandler(from, to), nil } if len(queueConf) == 0 { return nil, errors.New(`Missing required "queue" object`) } q, err := sorted.NewKeyValue(queueConf) if err != nil { return nil, err } isToIndex := false fromBs, err := ld.GetStorage(from) if err != nil { return nil, err } toBs, err := ld.GetStorage(to) if err != nil { return nil, err } if _, ok := fromBs.(*index.Index); !ok { if _, ok := toBs.(*index.Index); ok { isToIndex = true } } sh := newSyncHandler(from, to, fromBs, toBs, q) sh.toIndex = isToIndex sh.copierPoolSize = copierPoolSize if err := sh.readQueueToMemory(); err != nil { return nil, fmt.Errorf("Error reading sync queue to memory: %v", err) } if fullSync || blockFullSync { sh.logf("Doing full sync") didFullSync := make(chan bool, 1) go func() { for { n := sh.runSync("pending blobs queue", sh.enumeratePendingBlobs) if n > 0 { sh.logf("Queue sync copied %d blobs", n) continue } break } n := sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs)) sh.logf("Full sync copied %d blobs", n) didFullSync <- true sh.syncLoop() }() if blockFullSync { sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to) <-didFullSync sh.logf("Full sync complete.") } } else { go sh.syncLoop() } if validate { go sh.startFullValidation() } blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue) return sh, nil }
// GenerateClientConfig retuns a client configuration which can be used to // access a server defined by the provided low-level server configuration. func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) { missingConfig := func(param string) (*Config, error) { return nil, fmt.Errorf("required value for %q not found", param) } if serverConfig == nil { return nil, errors.New("server config is a required parameter") } param := "auth" auth := serverConfig.OptionalString(param, "") if auth == "" { return missingConfig(param) } listen := serverConfig.OptionalString("listen", "") baseURL := serverConfig.OptionalString("baseURL", "") if listen == "" { listen = baseURL } if listen == "" { return nil, errors.New("required value for 'listen' or 'baseURL' not found") } https := serverConfig.OptionalBool("https", false) if !strings.HasPrefix(listen, "http://") && !strings.HasPrefix(listen, "https://") { if !https { listen = "http://" + listen } else { listen = "https://" + listen } } httpsCert := serverConfig.OptionalString("httpsCert", "") // TODO(mpl): See if we can detect that the cert is not self-signed,and in // that case not add it to the trustedCerts var trustedList []string if https && httpsCert != "" { certPEMBlock, err := wkfs.ReadFile(httpsCert) if err != nil { return nil, fmt.Errorf("could not read certificate: %v", err) } sig, err := httputil.CertFingerprint(certPEMBlock) if err != nil { return nil, fmt.Errorf("could not get fingerprints of certificate: %v", err) } trustedList = []string{sig} } param = "prefixes" prefixes := serverConfig.OptionalObject(param) if len(prefixes) == 0 { return missingConfig(param) } param = "/sighelper/" sighelper := prefixes.OptionalObject(param) if len(sighelper) == 0 { return missingConfig(param) } param = "handlerArgs" handlerArgs := sighelper.OptionalObject(param) if len(handlerArgs) == 0 { return missingConfig(param) } param = "keyId" keyId := handlerArgs.OptionalString(param, "") if keyId == "" { return missingConfig(param) } param = "secretRing" secretRing := handlerArgs.OptionalString(param, "") if secretRing == "" { return missingConfig(param) } return &Config{ Servers: map[string]*Server{ "default": { Server: listen, Auth: auth, IsDefault: true, TrustedCerts: trustedList, }, }, Identity: keyId, IdentitySecretRing: secretRing, IgnoredFiles: []string{".DS_Store", "*~"}, }, nil }
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { ui := &UIHandler{ prefix: ld.MyPrefix(), sourceRoot: conf.OptionalString("sourceRoot", ""), resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes", constants.DefaultMaxResizeMem))), } cachePrefix := conf.OptionalString("cache", "") scaledImageConf := conf.OptionalObject("scaledImage") if err = conf.Validate(); err != nil { return } scaledImageKV, err := newKVOrNil(scaledImageConf) if err != nil { return nil, fmt.Errorf("in UI handler's scaledImage: %v", err) } if scaledImageKV != nil && cachePrefix == "" { return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache") } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err) } ui.Cache = bs ui.thumbMeta = NewThumbMeta(scaledImageKV) } if ui.sourceRoot == "" { ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT") if uistatic.IsAppEngine { if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot, filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil { hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+ " and the default sourceRoot dir %v does not exist or does not contain"+ " \"server/camlistored/ui/index.html\". devcam appengine can do that for you.", uistatic.GaeSourceRoot) log.Print(hint) return nil, errors.New("No sourceRoot found; UI not available.") } log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot) ui.sourceRoot = uistatic.GaeSourceRoot } if ui.sourceRoot == "" && uistatic.Files.IsEmpty() { ui.sourceRoot, err = osutil.GoPackagePath("camlistore.org") if err != nil { log.Printf("Warning: server not compiled with linked-in UI resources (HTML, JS, CSS), and camlistore.org not found in GOPATH.") } else { log.Printf("Using UI resources (HTML, JS, CSS) from disk, under %v", ui.sourceRoot) } } } if ui.sourceRoot != "" { ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui")) // Ignore any fileembed files: Files = &fileembed.Files{ DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")), } uistatic.Files = &fileembed.Files{ DirFallback: ui.uiDir, Listable: true, // In dev_appserver, allow edit-and-reload without // restarting. In production, though, it's faster to just // slurp it in. SlurpToMemory: uistatic.IsProdAppEngine, } } ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot) if err != nil { return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err) } if ui.sourceRoot != "" { ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "react"), "react.js") if err != nil { return nil, fmt.Errorf("Could not make react handler: %s", err) } ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "glitch"), "npc_piggy__x1_walk_png_1354829432.png") if err != nil { return nil, fmt.Errorf("Could not make glitch handler: %s", err) } ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "fontawesome"), "css/font-awesome.css") if err != nil { return nil, fmt.Errorf("Could not make fontawesome handler: %s", err) } ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "less"), "less.js") if err != nil { return nil, fmt.Errorf("Could not make less handler: %s", err) } } rootPrefix, _, err := ld.FindHandlerByType("root") if err != nil { return nil, errors.New("No root handler configured, which is necessary for the ui handler") } if h, err := ld.GetHandler(rootPrefix); err == nil { ui.root = h.(*RootHandler) ui.root.registerUIHandler(ui) } else { return nil, errors.New("failed to find the 'root' handler") } return ui, nil }