// NewHandler returns a Handler that proxies requests to an app. Start() on the // Handler starts the app. // The apiHost must end in a slash and is the camlistored API server for the app // process to hit. // The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted. // It must end in a slash, and be at minimum "/". // The conf object has the following members, related to the vars described in // doc/app-environment.txt: // "program", string, required. File name of the app's program executable. Either // an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH. // "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL. // "appConfig", object, optional. Additional configuration that the app can request from Camlistore. func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) { // TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available. name := conf.RequiredString("program") backendURL := conf.OptionalString("backendURL", "") appConfig := conf.OptionalObject("appConfig") // TODO(mpl): add an auth token in the extra config of the dev server config, // that the hello app can use to setup a status handler than only responds // to requests with that token. if err := conf.Validate(); err != nil { return nil, err } if apiHost == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name) } if appHandlerPrefix == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name) } if backendURL == "" { var err error // If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL // now (instead of letting the app itself do it), because we need to know it in advance in order // to set the app handler's proxy. backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix) if err != nil { return nil, err } } username, password := auth.RandToken(20), auth.RandToken(20) camliAuth := username + ":" + password basicAuth := auth.NewBasicAuth(username, password) envVars := map[string]string{ "CAMLI_API_HOST": apiHost, "CAMLI_AUTH": camliAuth, "CAMLI_APP_BACKEND_URL": backendURL, } if appConfig != nil { envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json" } proxyURL, err := url.Parse(backendURL) if err != nil { return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err) } return &Handler{ name: name, envVars: envVars, auth: basicAuth, appConfig: appConfig, proxy: httputil.NewSingleHostReverseProxy(proxyURL), backendURL: backendURL, }, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { var ( path = config.RequiredString("path") maxFileSize = config.OptionalInt("maxFileSize", 0) indexConf = config.OptionalObject("metaIndex") ) if err := config.Validate(); err != nil { return nil, err } return newStorage(path, int64(maxFileSize), indexConf) }
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) { h := &Host{ baseURL: ld.BaseURL(), importerBase: ld.BaseURL() + ld.MyPrefix(), imp: make(map[string]*importer), } for k, impl := range importers { h.importers = append(h.importers, k) var clientID, clientSecret string if impConf := cfg.OptionalObject(k); impConf != nil { clientID = impConf.OptionalString("clientID", "") clientSecret = impConf.OptionalString("clientSecret", "") // Special case: allow clientSecret to be of form "clientID:clientSecret" // if the clientID is empty. if clientID == "" && strings.Contains(clientSecret, ":") { if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 { clientID, clientSecret = f[0], f[1] } } if err := impConf.Validate(); err != nil { return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err) } } if clientSecret != "" && clientID == "" { return nil, fmt.Errorf("Invalid static configuration for importer %q: clientSecret specified without clientID", k) } imp := &importer{ host: h, name: k, impl: impl, clientID: clientID, clientSecret: clientSecret, } h.imp[k] = imp } if err := cfg.Validate(); err != nil { return nil, err } sort.Strings(h.importers) return h, nil }
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) { hc := HostConfig{ BaseURL: ld.BaseURL(), Prefix: ld.MyPrefix(), } ClientId := make(map[string]string) ClientSecret := make(map[string]string) for k, _ := range importers { var clientId, clientSecret string if impConf := cfg.OptionalObject(k); impConf != nil { clientId = impConf.OptionalString("clientID", "") clientSecret = impConf.OptionalString("clientSecret", "") // Special case: allow clientSecret to be of form "clientId:clientSecret" // if the clientId is empty. if clientId == "" && strings.Contains(clientSecret, ":") { if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 { clientId, clientSecret = f[0], f[1] } } if err := impConf.Validate(); err != nil { return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err) } ClientId[k] = clientId ClientSecret[k] = clientSecret } } if err := cfg.Validate(); err != nil { return nil, err } hc.ClientId = ClientId hc.ClientSecret = ClientSecret host, err := NewHost(hc) if err != nil { return nil, err } host.didInit.Add(1) return host, nil }
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { ui := &UIHandler{ prefix: ld.MyPrefix(), JSONSignRoot: conf.OptionalString("jsonSignRoot", ""), sourceRoot: conf.OptionalString("sourceRoot", ""), resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes", constants.DefaultMaxResizeMem))), } pubRoots := conf.OptionalList("publishRoots") cachePrefix := conf.OptionalString("cache", "") scaledImageConf := conf.OptionalObject("scaledImage") if err = conf.Validate(); err != nil { return } if ui.JSONSignRoot != "" { h, _ := ld.GetHandler(ui.JSONSignRoot) if sigh, ok := h.(*signhandler.Handler); ok { ui.sigh = sigh } } if os.Getenv("CAMLI_PUBLISH_ENABLED") == "false" { // Hack for dev server, to simplify its config with devcam server --publish=false. pubRoots = nil } ui.publishRoots = make(map[string]*PublishHandler) for _, pubRoot := range pubRoots { h, err := ld.GetHandler(pubRoot) if err != nil { return nil, fmt.Errorf("UI handler's publishRoots references invalid %q", pubRoot) } pubh, ok := h.(*PublishHandler) if !ok { return nil, fmt.Errorf("UI handler's publishRoots references invalid %q; not a PublishHandler", pubRoot) } ui.publishRoots[pubRoot] = pubh } checkType := func(key string, htype string) { v := conf.OptionalString(key, "") if v == "" { return } ct := ld.GetHandlerType(v) if ct == "" { err = fmt.Errorf("UI handler's %q references non-existant %q", key, v) } else if ct != htype { err = fmt.Errorf("UI handler's %q references %q of type %q; expected type %q", key, v, ct, htype) } } checkType("searchRoot", "search") checkType("jsonSignRoot", "jsonsign") if err != nil { return } scaledImageKV, err := newKVOrNil(scaledImageConf) if err != nil { return nil, fmt.Errorf("in UI handler's scaledImage: %v", err) } if scaledImageKV != nil && cachePrefix == "" { return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache") } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err) } ui.Cache = bs ui.thumbMeta = newThumbMeta(scaledImageKV) } if ui.sourceRoot == "" { ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT") if uistatic.IsAppEngine { if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot, filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil { hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+ " and the default sourceRoot dir %v does not exist or does not contain"+ " \"server/camlistored/ui/index.html\". devcam appengine can do that for you.", uistatic.GaeSourceRoot) log.Print(hint) return nil, errors.New("No sourceRoot found; UI not available.") } log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot) ui.sourceRoot = uistatic.GaeSourceRoot } } if ui.sourceRoot != "" { ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui")) // Ignore any fileembed files: Files = &fileembed.Files{ DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")), } uistatic.Files = &fileembed.Files{ DirFallback: ui.uiDir, Listable: true, // In dev_appserver, allow edit-and-reload without // restarting. In production, though, it's faster to just // slurp it in. SlurpToMemory: uistatic.IsProdAppEngine, } } ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot) if err != nil { return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err) } if ui.sourceRoot != "" { ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "react"), "react.js") if err != nil { return nil, fmt.Errorf("Could not make react handler: %s", err) } ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "glitch"), "npc_piggy__x1_walk_png_1354829432.png") if err != nil { return nil, fmt.Errorf("Could not make glitch handler: %s", err) } } rootPrefix, _, err := ld.FindHandlerByType("root") if err != nil { return nil, errors.New("No root handler configured, which is necessary for the ui handler") } if h, err := ld.GetHandler(rootPrefix); err == nil { ui.root = h.(*RootHandler) ui.root.registerUIHandler(ui) } else { return nil, errors.New("failed to find the 'root' handler") } return ui, nil }
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { var ( from = conf.RequiredString("from") to = conf.RequiredString("to") fullSync = conf.OptionalBool("fullSyncOnStart", false) blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false) idle = conf.OptionalBool("idle", false) queueConf = conf.OptionalObject("queue") copierPoolSize = conf.OptionalInt("copierPoolSize", 5) validate = conf.OptionalBool("validateOnStart", validateOnStartDefault) ) if err := conf.Validate(); err != nil { return nil, err } if idle { return newIdleSyncHandler(from, to), nil } if len(queueConf) == 0 { return nil, errors.New(`Missing required "queue" object`) } q, err := sorted.NewKeyValue(queueConf) if err != nil { return nil, err } isToIndex := false fromBs, err := ld.GetStorage(from) if err != nil { return nil, err } toBs, err := ld.GetStorage(to) if err != nil { return nil, err } if _, ok := fromBs.(*index.Index); !ok { if _, ok := toBs.(*index.Index); ok { isToIndex = true } } sh := newSyncHandler(from, to, fromBs, toBs, q) sh.toIndex = isToIndex sh.copierPoolSize = copierPoolSize if err := sh.readQueueToMemory(); err != nil { return nil, fmt.Errorf("Error reading sync queue to memory: %v", err) } if fullSync || blockFullSync { sh.logf("Doing full sync") didFullSync := make(chan bool, 1) go func() { for { n := sh.runSync("queue", sh.enumeratePendingBlobs) if n > 0 { sh.logf("Queue sync copied %d blobs", n) continue } break } n := sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs)) sh.logf("Full sync copied %d blobs", n) didFullSync <- true sh.syncLoop() }() if blockFullSync { sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to) <-didFullSync sh.logf("Full sync complete.") } } else { go sh.syncLoop() } if validate { go sh.startFullValidation() } blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue) return sh, nil }
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { var ( from = conf.RequiredString("from") to = conf.RequiredString("to") fullSync = conf.OptionalBool("fullSyncOnStart", false) blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false) idle = conf.OptionalBool("idle", false) queueConf = conf.OptionalObject("queue") ) if err := conf.Validate(); err != nil { return nil, err } if idle { synch, err := createIdleSyncHandler(from, to) if err != nil { return nil, err } return synch, nil } if len(queueConf) == 0 { return nil, errors.New(`Missing required "queue" object`) } q, err := sorted.NewKeyValue(queueConf) if err != nil { return nil, err } isToIndex := false fromBs, err := ld.GetStorage(from) if err != nil { return nil, err } toBs, err := ld.GetStorage(to) if err != nil { return nil, err } if _, ok := fromBs.(*index.Index); !ok { if _, ok := toBs.(*index.Index); ok { isToIndex = true } } sh, err := createSyncHandler(from, to, fromBs, toBs, q, isToIndex) if err != nil { return nil, err } if fullSync || blockFullSync { didFullSync := make(chan bool, 1) go func() { n := sh.runSync("queue", sh.enumerateQueuedBlobs) sh.logf("Queue sync copied %d blobs", n) n = sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs)) sh.logf("Full sync copied %d blobs", n) didFullSync <- true sh.syncQueueLoop() }() if blockFullSync { sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to) <-didFullSync sh.logf("Full sync complete.") } } else { go sh.syncQueueLoop() } blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue) return sh, nil }
func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { ph := &PublishHandler{ handlerFinder: ld, } ph.RootName = conf.RequiredString("rootName") jsFiles := conf.OptionalList("js") ph.CSSFiles = conf.OptionalList("css") goTemplateFile := conf.RequiredString("goTemplate") blobRoot := conf.RequiredString("blobRoot") searchRoot := conf.RequiredString("searchRoot") cachePrefix := conf.OptionalString("cache", "") scaledImageConf := conf.OptionalObject("scaledImage") bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "") rootNode := conf.OptionalList("rootPermanode") ph.sourceRoot = conf.OptionalString("sourceRoot", "") if err = conf.Validate(); err != nil { return } if ph.RootName == "" { return nil, errors.New("invalid empty rootName") } bs, err := ld.GetStorage(blobRoot) if err != nil { return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err) } ph.Storage = bs si, err := ld.GetHandler(searchRoot) if err != nil { return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err) } var ok bool ph.Search, ok = si.(*search.Handler) if !ok { return nil, fmt.Errorf("publish handler's searchRoot of %q is of type %T, expecting a search handler", searchRoot, si) } if rootNode != nil { if len(rootNode) != 2 { return nil, fmt.Errorf("rootPermanode config must contain the jsonSignerHandler and the permanode hash") } if t := ld.GetHandlerType(rootNode[0]); t != "jsonsign" { return nil, fmt.Errorf("publish handler's rootPermanode first value not a jsonsign") } h, _ := ld.GetHandler(rootNode[0]) jsonSign := h.(*signhandler.Handler) pn, ok := blob.Parse(rootNode[1]) if !ok { return nil, fmt.Errorf("Invalid \"rootPermanode\" value; was expecting a blobRef, got %q.", rootNode[1]) } if err := ph.setRootNode(jsonSign, pn); err != nil { return nil, fmt.Errorf("error setting publish root permanode: %v", err) } } else { if bootstrapSignRoot != "" { if t := ld.GetHandlerType(bootstrapSignRoot); t != "jsonsign" { return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign") } h, _ := ld.GetHandler(bootstrapSignRoot) jsonSign := h.(*signhandler.Handler) if err := ph.bootstrapPermanode(jsonSign); err != nil { return nil, fmt.Errorf("error bootstrapping permanode: %v", err) } } } scaledImageKV, err := newKVOrNil(scaledImageConf) if err != nil { return nil, fmt.Errorf("in publish handler's scaledImage: %v", err) } if scaledImageKV != nil && cachePrefix == "" { return nil, fmt.Errorf("in publish handler, can't specify scaledImage without cache") } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err) } ph.Cache = bs ph.thumbMeta = newThumbMeta(scaledImageKV) } // TODO(mpl): check that it works on appengine too. if ph.sourceRoot == "" { ph.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT") } if ph.sourceRoot != "" { ph.uiDir = filepath.Join(ph.sourceRoot, filepath.FromSlash("server/camlistored/ui")) // Ignore any fileembed files: Files = &fileembed.Files{ DirFallback: filepath.Join(ph.sourceRoot, filepath.FromSlash("pkg/server")), } uistatic.Files = &fileembed.Files{ DirFallback: ph.uiDir, } } ph.closureHandler, err = ph.makeClosureHandler(ph.sourceRoot) if err != nil { return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ph.sourceRoot, err) } ph.goTemplate, err = goTemplate(goTemplateFile) if err != nil { return nil, err } ph.setClosureName(jsFiles) return ph, nil }
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { ui := &UIHandler{ prefix: ld.MyPrefix(), sourceRoot: conf.OptionalString("sourceRoot", ""), resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes", constants.DefaultMaxResizeMem))), } cachePrefix := conf.OptionalString("cache", "") scaledImageConf := conf.OptionalObject("scaledImage") if err = conf.Validate(); err != nil { return } scaledImageKV, err := newKVOrNil(scaledImageConf) if err != nil { return nil, fmt.Errorf("in UI handler's scaledImage: %v", err) } if scaledImageKV != nil && cachePrefix == "" { return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache") } if cachePrefix != "" { bs, err := ld.GetStorage(cachePrefix) if err != nil { return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err) } ui.Cache = bs ui.thumbMeta = NewThumbMeta(scaledImageKV) } if ui.sourceRoot == "" { ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT") if uistatic.IsAppEngine { if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot, filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil { hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+ " and the default sourceRoot dir %v does not exist or does not contain"+ " \"server/camlistored/ui/index.html\". devcam appengine can do that for you.", uistatic.GaeSourceRoot) log.Print(hint) return nil, errors.New("No sourceRoot found; UI not available.") } log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot) ui.sourceRoot = uistatic.GaeSourceRoot } } if ui.sourceRoot != "" { ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui")) // Ignore any fileembed files: Files = &fileembed.Files{ DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")), } uistatic.Files = &fileembed.Files{ DirFallback: ui.uiDir, Listable: true, // In dev_appserver, allow edit-and-reload without // restarting. In production, though, it's faster to just // slurp it in. SlurpToMemory: uistatic.IsProdAppEngine, } } ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot) if err != nil { return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err) } if ui.sourceRoot != "" { ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "react"), "react.js") if err != nil { return nil, fmt.Errorf("Could not make react handler: %s", err) } ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "glitch"), "npc_piggy__x1_walk_png_1354829432.png") if err != nil { return nil, fmt.Errorf("Could not make glitch handler: %s", err) } ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "fontawesome"), "css/font-awesome.css") if err != nil { return nil, fmt.Errorf("Could not make fontawesome handler: %s", err) } ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join("third_party", "less"), "less.js") if err != nil { return nil, fmt.Errorf("Could not make less handler: %s", err) } } rootPrefix, _, err := ld.FindHandlerByType("root") if err != nil { return nil, errors.New("No root handler configured, which is necessary for the ui handler") } if h, err := ld.GetHandler(rootPrefix); err == nil { ui.root = h.(*RootHandler) ui.root.registerUIHandler(ui) } else { return nil, errors.New("failed to find the 'root' handler") } return ui, nil }
// GenerateClientConfig retuns a client configuration which can be used to // access a server defined by the provided low-level server configuration. func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) { missingConfig := func(param string) (*Config, error) { return nil, fmt.Errorf("required value for '%s' not found", param) } if serverConfig == nil { return nil, errors.New("server config is a required parameter") } param := "auth" auth := serverConfig.OptionalString(param, "") if auth == "" { return missingConfig(param) } listen := serverConfig.OptionalString("listen", "") baseURL := serverConfig.OptionalString("baseURL", "") if listen == "" { listen = baseURL } if listen == "" { return nil, errors.New("required value for 'listen' or 'baseURL' not found") } https := serverConfig.OptionalBool("https", false) if !strings.HasPrefix(listen, "http://") && !strings.HasPrefix(listen, "https://") { if !https { listen = "http://" + listen } else { listen = "https://" + listen } } param = "httpsCert" httpsCert := serverConfig.OptionalString(param, "") if https && httpsCert == "" { return missingConfig(param) } // TODO(mpl): See if we can detect that the cert is not self-signed,and in // that case not add it to the trustedCerts var trustedList []string if https && httpsCert != "" { certPEMBlock, err := wkfs.ReadFile(httpsCert) if err != nil { return nil, fmt.Errorf("could not read certificate: %v", err) } sig, err := httputil.CertFingerprint(certPEMBlock) if err != nil { return nil, fmt.Errorf("could not get fingerprints of certificate: %v", err) } trustedList = []string{sig} } param = "prefixes" prefixes := serverConfig.OptionalObject(param) if len(prefixes) == 0 { return missingConfig(param) } param = "/sighelper/" sighelper := prefixes.OptionalObject(param) if len(sighelper) == 0 { return missingConfig(param) } param = "handlerArgs" handlerArgs := sighelper.OptionalObject(param) if len(handlerArgs) == 0 { return missingConfig(param) } param = "keyId" keyId := handlerArgs.OptionalString(param, "") if keyId == "" { return missingConfig(param) } param = "secretRing" secretRing := handlerArgs.OptionalString(param, "") if secretRing == "" { return missingConfig(param) } return &Config{ Servers: map[string]*Server{ "default": { Server: listen, Auth: auth, IsDefault: true, TrustedCerts: trustedList, }, }, Identity: keyId, IdentitySecretRing: secretRing, IgnoredFiles: []string{".DS_Store"}, }, nil }