Example #1
0
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &condStorage{}

	receive := conf.OptionalStringOrObject("write")
	read := conf.RequiredString("read")
	remove := conf.OptionalString("remove", "")
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if receive != nil {
		sto.storageForReceive, err = buildStorageForReceive(ld, receive)
		if err != nil {
			return
		}
	}

	sto.read, err = ld.GetStorage(read)
	if err != nil {
		return
	}

	if remove != "" {
		sto.remove, err = ld.GetStorage(remove)
		if err != nil {
			return
		}
	}
	return sto, nil
}
Example #2
0
func indexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
	is := &indexStorage{}
	var (
		blobPrefix = config.RequiredString("blobSource")
		ns         = config.OptionalString("namespace", "")
	)
	if err := config.Validate(); err != nil {
		return nil, err
	}
	sto, err := ld.GetStorage(blobPrefix)
	if err != nil {
		return nil, err
	}
	is.ns, err = sanitizeNamespace(ns)
	if err != nil {
		return nil, err
	}

	ix, err := index.New(is)
	if err != nil {
		return nil, err
	}
	ix.BlobSource = sto
	ix.KeyFetcher = ix.BlobSource // TODO(bradfitz): global search? something else?
	return ix, nil
}
Example #3
0
// detectConfigChange returns an informative error if conf contains obsolete keys.
func detectConfigChange(conf jsonconfig.Obj) error {
	oldHTTPSKey, oldHTTPSCert := conf.OptionalString("HTTPSKeyFile", ""), conf.OptionalString("HTTPSCertFile", "")
	if oldHTTPSKey != "" || oldHTTPSCert != "" {
		return fmt.Errorf("Config keys %q and %q have respectively been renamed to %q and %q, please fix your server config.",
			"HTTPSKeyFile", "HTTPSCertFile", "httpsKey", "httpsCert")
	}
	return nil
}
Example #4
0
// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
	// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
	name := conf.RequiredString("program")
	backendURL := conf.OptionalString("backendURL", "")
	appConfig := conf.OptionalObject("appConfig")
	// TODO(mpl): add an auth token in the extra config of the dev server config,
	// that the hello app can use to setup a status handler than only responds
	// to requests with that token.
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if apiHost == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
	}
	if appHandlerPrefix == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
	}

	if backendURL == "" {
		var err error
		// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
		// now (instead of letting the app itself do it), because we need to know it in advance in order
		// to set the app handler's proxy.
		backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
		if err != nil {
			return nil, err
		}
	}

	username, password := auth.RandToken(20), auth.RandToken(20)
	camliAuth := username + ":" + password
	basicAuth := auth.NewBasicAuth(username, password)
	envVars := map[string]string{
		"CAMLI_API_HOST":        apiHost,
		"CAMLI_AUTH":            camliAuth,
		"CAMLI_APP_BACKEND_URL": backendURL,
	}
	if appConfig != nil {
		envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
	}

	proxyURL, err := url.Parse(backendURL)
	if err != nil {
		return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
	}
	return &Handler{
		name:       name,
		envVars:    envVars,
		auth:       basicAuth,
		appConfig:  appConfig,
		proxy:      httputil.NewSingleHostReverseProxy(proxyURL),
		backendURL: backendURL,
	}, nil
}
Example #5
0
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
	sto := &appengineStorage{
		namespace: config.OptionalString("namespace", ""),
	}
	if err := config.Validate(); err != nil {
		return nil, err
	}
	sto.namespace, err = sanitizeNamespace(sto.namespace)
	if err != nil {
		return nil, err
	}
	return sto, nil
}
Example #6
0
// convertToMultiServers takes an old style single-server client configuration and maps it to new a multi-servers configuration that is returned.
func convertToMultiServers(conf jsonconfig.Obj) (jsonconfig.Obj, error) {
	server := conf.OptionalString("server", "")
	if server == "" {
		return nil, errors.New("Could not convert config to multi-servers style: no \"server\" key found.")
	}
	newConf := jsonconfig.Obj{
		"servers": map[string]interface{}{
			"server": map[string]interface{}{
				"auth":    conf.OptionalString("auth", ""),
				"default": true,
				"server":  server,
			},
		},
		"identity":           conf.OptionalString("identity", ""),
		"identitySecretRing": conf.OptionalString("identitySecretRing", ""),
	}
	if ignoredFiles := conf.OptionalList("ignoredFiles"); ignoredFiles != nil {
		var list []interface{}
		for _, v := range ignoredFiles {
			list = append(list, v)
		}
		newConf["ignoredFiles"] = list
	}
	return newConf, nil
}
Example #7
0
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
	var (
		// either a short form ("26F5ABDA") or one the longer forms.
		keyId = conf.RequiredString("keyId")

		pubKeyDestPrefix = conf.OptionalString("publicKeyDest", "")
		secretRing       = conf.OptionalString("secretRing", "")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	h := &Handler{
		secretRing: secretRing,
	}

	var err error
	h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
	if err != nil {
		return nil, err
	}

	h.pubKey, err = jsonsign.ArmoredPublicKey(h.entity)

	ms := &memory.Storage{}
	h.pubKeyBlobRef = blob.SHA1FromString(h.pubKey)
	if _, err := ms.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(h.pubKey)); err != nil {
		return nil, fmt.Errorf("could not store pub key blob: %v", err)
	}
	h.pubKeyFetcher = ms

	if pubKeyDestPrefix != "" {
		sto, err := ld.GetStorage(pubKeyDestPrefix)
		if err != nil {
			return nil, err
		}
		h.pubKeyDest = sto
	}
	h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
	h.pubKeyHandler = &gethandler.Handler{
		Fetcher: ms,
	}

	h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(h.pubKey), h.entity)
	if err != nil {
		return nil, err
	}

	return h, nil
}
Example #8
0
func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
	indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
	ownerBlobStr := conf.RequiredString("owner")
	devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "")
	slurpToMemory := conf.OptionalBool("slurpToMemory", false)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if devBlockStartupPrefix != "" {
		_, err := ld.GetHandler(devBlockStartupPrefix)
		if err != nil {
			return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err)
		}
	}

	indexHandler, err := ld.GetHandler(indexPrefix)
	if err != nil {
		return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
	}
	indexer, ok := indexHandler.(index.Interface)
	if !ok {
		return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
	}
	ownerBlobRef, ok := blob.Parse(ownerBlobStr)
	if !ok {
		return nil, fmt.Errorf("search 'owner' has malformed blobref %q; expecting e.g. sha1-xxxxxxxxxxxx",
			ownerBlobStr)
	}
	h := NewHandler(indexer, ownerBlobRef)
	if slurpToMemory {
		ii := indexer.(*index.Index)
		ii.Lock()
		corpus, err := ii.KeepInMemory()
		if err != nil {
			ii.Unlock()
			return nil, fmt.Errorf("error slurping index to memory: %v", err)
		}
		h.SetCorpus(corpus)
		ii.Unlock()
	}
	return h, nil
}
Example #9
0
// FromJSONConfig creates an HandlerConfig from the contents of config.
// serverBaseURL is used if it is not found in config.
func FromJSONConfig(config jsonconfig.Obj, serverBaseURL string) (HandlerConfig, error) {
	hc := HandlerConfig{
		Program:       config.RequiredString("program"),
		Prefix:        config.RequiredString("prefix"),
		BackendURL:    config.OptionalString("backendURL", ""),
		Listen:        config.OptionalString("listen", ""),
		APIHost:       config.OptionalString("apiHost", ""),
		ServerListen:  config.OptionalString("serverListen", ""),
		ServerBaseURL: config.OptionalString("serverBaseURL", ""),
		AppConfig:     config.OptionalObject("appConfig"),
	}
	if hc.ServerBaseURL == "" {
		hc.ServerBaseURL = serverBaseURL
	}
	if err := config.Validate(); err != nil {
		return HandlerConfig{}, err
	}
	return hc, nil
}
Example #10
0
// ConfigFromJSON populates Config from cfg, and validates
// cfg. It returns an error if cfg fails to validate.
func configFromJSON(cfg jsonconfig.Obj) (config, error) {
	conf := config{
		server:     cfg.OptionalString("host", "localhost"),
		database:   cfg.RequiredString("database"),
		collection: cfg.OptionalString("collection", "blobs"),
		user:       cfg.OptionalString("user", ""),
		password:   cfg.OptionalString("password", ""),
	}
	if err := cfg.Validate(); err != nil {
		return config{}, err
	}
	return conf, nil
}
Example #11
0
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
	conninfo := fmt.Sprintf("user=%s dbname=%s host=%s password=%s sslmode=%s",
		cfg.RequiredString("user"),
		cfg.RequiredString("database"),
		cfg.OptionalString("host", "localhost"),
		cfg.OptionalString("password", ""),
		cfg.OptionalString("sslmode", "require"),
	)
	if err := cfg.Validate(); err != nil {
		return nil, err
	}
	db, err := sql.Open("postgres", conninfo)
	if err != nil {
		return nil, err
	}
	for _, tableSql := range SQLCreateTables() {
		if _, err := db.Exec(tableSql); err != nil {
			return nil, fmt.Errorf("error creating table with %q: %v", tableSql, err)
		}
	}
	for _, statement := range SQLDefineReplace() {
		if _, err := db.Exec(statement); err != nil {
			return nil, fmt.Errorf("error setting up replace statement with %q: %v", statement, err)
		}
	}
	r, err := db.Query(fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, SchemaVersion()))
	if err != nil {
		return nil, fmt.Errorf("error setting schema version: %v", err)
	}
	r.Close()

	kv := &keyValue{
		db: db,
		KeyValue: &sqlkv.KeyValue{
			DB:              db,
			SetFunc:         altSet,
			BatchSetFunc:    altBatchSet,
			PlaceHolderFunc: replacePlaceHolders,
		},
	}
	if err := kv.ping(); err != nil {
		return nil, fmt.Errorf("PostgreSQL db unreachable: %v", err)
	}
	version, err := kv.SchemaVersion()
	if err != nil {
		return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
	}
	if version != requiredSchemaVersion {
		if env.IsDev() {
			// Good signal that we're using the devcam server, so help out
			// the user with a more useful tip:
			return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
		}
		return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
			version, requiredSchemaVersion)
	}

	return kv, nil
}
Example #12
0
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
	ins := &instance{
		server:   cfg.OptionalString("host", "localhost"),
		database: cfg.RequiredString("database"),
		user:     cfg.OptionalString("user", ""),
		password: cfg.OptionalString("password", ""),
	}
	if err := cfg.Validate(); err != nil {
		return nil, err
	}
	db, err := ins.getCollection()
	if err != nil {
		return nil, err
	}
	return &keyValue{db: db, session: ins.session}, nil
}
Example #13
0
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.Storage, err error) {
	metaConf := config.RequiredObject("metaIndex")
	sto := &storage{}
	agreement := config.OptionalString("I_AGREE", "")
	const wantAgreement = "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change."
	if agreement != wantAgreement {
		return nil, errors.New("Use of the 'encrypt' target without the proper I_AGREE value.")
	}

	key := config.OptionalString("key", "")
	keyFile := config.OptionalString("keyFile", "")
	var keyb []byte
	switch {
	case key != "":
		keyb, err = hex.DecodeString(key)
		if err != nil || len(keyb) != 16 {
			return nil, fmt.Errorf("The 'key' parameter must be 16 bytes of 32 hex digits. (currently fixed at AES-128)")
		}
	case keyFile != "":
		// TODO: check that keyFile's unix permissions aren't too permissive.
		keyb, err = ioutil.ReadFile(keyFile)
		if err != nil {
			return nil, fmt.Errorf("Reading key file %v: %v", keyFile, err)
		}
	}
	blobStorage := config.RequiredString("blobs")
	metaStorage := config.RequiredString("meta")
	if err := config.Validate(); err != nil {
		return nil, err
	}

	sto.index, err = sorted.NewKeyValue(metaConf)
	if err != nil {
		return
	}

	sto.blobs, err = ld.GetStorage(blobStorage)
	if err != nil {
		return
	}
	sto.meta, err = ld.GetStorage(metaStorage)
	if err != nil {
		return
	}

	if keyb == nil {
		// TODO: add a way to prompt from stdin on start? or keychain support?
		return nil, errors.New("no encryption key set with 'key' or 'keyFile'")
	}

	if err := sto.setKey(keyb); err != nil {
		return nil, err
	}

	start := time.Now()
	log.Printf("Reading encryption metadata...")
	if err := sto.readAllMetaBlobs(); err != nil {
		return nil, fmt.Errorf("Error scanning metadata on start-up: %v", err)
	}
	log.Printf("Read all encryption metadata in %.3f seconds", time.Since(start).Seconds())

	return sto, nil
}
Example #14
0
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
	hostname := config.OptionalString("hostname", "s3.amazonaws.com")
	cacheSize := config.OptionalInt64("cacheSize", 32<<20)
	client := &s3.Client{
		Auth: &s3.Auth{
			AccessKey:       config.RequiredString("aws_access_key"),
			SecretAccessKey: config.RequiredString("aws_secret_access_key"),
			Hostname:        hostname,
		},
		PutGate: syncutil.NewGate(maxParallelHTTP),
	}
	bucket := config.RequiredString("bucket")
	var dirPrefix string
	if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 {
		dirPrefix = parts[1]
		bucket = parts[0]
	}
	if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") {
		dirPrefix += "/"
	}
	sto := &s3Storage{
		s3Client:  client,
		bucket:    bucket,
		dirPrefix: dirPrefix,
		hostname:  hostname,
	}
	skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
	if err := config.Validate(); err != nil {
		return nil, err
	}
	if cacheSize != 0 {
		sto.cache = memory.NewCache(cacheSize)
	}
	if !skipStartupCheck {
		_, err := client.ListBucket(sto.bucket, "", 1)
		if serr, ok := err.(*s3.Error); ok {
			if serr.AmazonCode == "NoSuchBucket" {
				return nil, fmt.Errorf("Bucket %q doesn't exist.", sto.bucket)
			}

			// This code appears when the hostname has dots in it:
			if serr.AmazonCode == "PermanentRedirect" {
				loc, lerr := client.BucketLocation(sto.bucket)
				if lerr != nil {
					return nil, fmt.Errorf("Wrong server for bucket %q; and error determining bucket's location: %v", sto.bucket, lerr)
				}
				client.Auth.Hostname = loc
				_, err = client.ListBucket(sto.bucket, "", 1)
				if err == nil {
					log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
				}
			}

			// This path occurs when the user set the
			// wrong server, or didn't set one at all, but
			// the bucket doesn't have dots in it:
			if serr.UseEndpoint != "" {
				// UseEndpoint will be e.g. "brads3test-ca.s3-us-west-1.amazonaws.com"
				// But we only want the "s3-us-west-1.amazonaws.com" part.
				client.Auth.Hostname = strings.TrimPrefix(serr.UseEndpoint, sto.bucket+".")
				_, err = client.ListBucket(sto.bucket, "", 1)
				if err == nil {
					log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
				}
			}
		}
		if err != nil {
			return nil, fmt.Errorf("Error listing bucket %s: %v", sto.bucket, err)
		}
	}
	return sto, nil
}
Example #15
0
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
	var (
		user     = cfg.RequiredString("user")
		database = cfg.RequiredString("database")
		host     = cfg.OptionalString("host", "")
		password = cfg.OptionalString("password", "")
	)
	if err := cfg.Validate(); err != nil {
		return nil, err
	}
	var err error
	if host != "" {
		host, err = maybeRemapCloudSQL(host)
		if err != nil {
			return nil, err
		}
		if !strings.Contains(host, ":") {
			host += ":3306"
		}
		host = "tcp(" + host + ")"
	}
	// The DSN does NOT have a database name in it so it's
	// cacheable and can be shared between different queues & the
	// index, all sharing the same database server, cutting down
	// number of TCP connections required. We add the database
	// name in queries instead.
	dsn := fmt.Sprintf("%s:%s@%s/", user, password, host)

	db, err := openOrCachedDB(dsn)
	if err != nil {
		return nil, err
	}

	if err := CreateDB(db, database); err != nil {
		return nil, err
	}
	for _, tableSQL := range SQLCreateTables() {
		tableSQL = strings.Replace(tableSQL, "/*DB*/", database, -1)
		if _, err := db.Exec(tableSQL); err != nil {
			errMsg := "error creating table with %q: %v."
			createError := err
			sv, err := serverVersion(db)
			if err != nil {
				return nil, err
			}
			if !hasLargeVarchar(sv) {

				errMsg += "\nYour MySQL server is too old (< 5.0.3) to support VARCHAR larger than 255."
			}
			return nil, fmt.Errorf(errMsg, tableSQL, createError)
		}
	}
	if _, err := db.Exec(fmt.Sprintf(`REPLACE INTO %s.meta VALUES ('version', '%d')`, database, SchemaVersion())); err != nil {
		return nil, fmt.Errorf("error setting schema version: %v", err)
	}

	kv := &keyValue{
		dsn: dsn,
		db:  db,
		KeyValue: &sqlkv.KeyValue{
			DB:          db,
			TablePrefix: database + ".",
			Gate:        syncutil.NewGate(20), // arbitrary limit. TODO: configurable, automatically-learned?
		},
	}
	if err := kv.ping(); err != nil {
		return nil, fmt.Errorf("MySQL db unreachable: %v", err)
	}
	version, err := kv.SchemaVersion()
	if err != nil {
		return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
	}
	if version != requiredSchemaVersion {
		if version == 20 && requiredSchemaVersion == 21 {
			fmt.Fprintf(os.Stderr, fixSchema20to21)
		}
		if env.IsDev() {
			// Good signal that we're using the devcam server, so help out
			// the user with a more useful tip:
			return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
		}
		return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
			version, requiredSchemaVersion)
	}

	return kv, nil
}
Example #16
0
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
	var (
		user     = cfg.RequiredString("user")
		database = cfg.RequiredString("database")
		host     = cfg.OptionalString("host", "")
		password = cfg.OptionalString("password", "")
	)
	if err := cfg.Validate(); err != nil {
		return nil, err
	}
	if !validDatabaseName(database) {
		return nil, fmt.Errorf("%q looks like an invalid database name", database)
	}
	var err error
	if host != "" {
		host, err = maybeRemapCloudSQL(host)
		if err != nil {
			return nil, err
		}
		if !strings.Contains(host, ":") {
			host += ":3306"
		}
		host = "tcp(" + host + ")"
	}
	// The DSN does NOT have a database name in it so it's
	// cacheable and can be shared between different queues & the
	// index, all sharing the same database server, cutting down
	// number of TCP connections required. We add the database
	// name in queries instead.
	dsn := fmt.Sprintf("%s:%s@%s/", user, password, host)

	db, err := openOrCachedDB(dsn)
	if err != nil {
		return nil, err
	}

	if err := CreateDB(db, database); err != nil {
		return nil, err
	}
	if err := createTables(db, database); err != nil {
		return nil, err
	}

	kv := &keyValue{
		dsn: dsn,
		db:  db,
		KeyValue: &sqlkv.KeyValue{
			DB:          db,
			TablePrefix: database + ".",
			Gate:        syncutil.NewGate(20), // arbitrary limit. TODO: configurable, automatically-learned?
		},
	}
	if err := kv.ping(); err != nil {
		return nil, fmt.Errorf("MySQL db unreachable: %v", err)
	}

	version, err := kv.SchemaVersion()
	if err != nil {
		return nil, fmt.Errorf("error getting current database schema version: %v", err)
	}
	if version == 0 {
		// Newly created table case
		if _, err := db.Exec(fmt.Sprintf(`REPLACE INTO %s.meta VALUES ('version', ?)`, database), requiredSchemaVersion); err != nil {
			return nil, fmt.Errorf("error setting schema version: %v", err)
		}
		return kv, nil
	}
	if version != requiredSchemaVersion {
		if version == 20 && requiredSchemaVersion == 21 {
			fmt.Fprintf(os.Stderr, fixSchema20to21)
		}
		if env.IsDev() {
			// Good signal that we're using the devcam server, so help out
			// the user with a more useful tip:
			return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
		}
		return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
			version, requiredSchemaVersion)
	}

	return kv, nil
}
Example #17
0
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	checkType := func(key string, htype string) {
		v := conf.OptionalString(key, "")
		if v == "" {
			return
		}
		ct := ld.GetHandlerType(v)
		if ct == "" {
			err = fmt.Errorf("root handler's %q references non-existant %q", key, v)
		} else if ct != htype {
			err = fmt.Errorf("root handler's %q references %q of type %q; expected type %q", key, v, ct, htype)
		}
	}
	checkType("searchRoot", "search")
	checkType("jsonSignRoot", "jsonsign")
	if err != nil {
		return
	}
	username, _ := getUserName()
	root := &RootHandler{
		BlobRoot:     conf.OptionalString("blobRoot", ""),
		SearchRoot:   conf.OptionalString("searchRoot", ""),
		JSONSignRoot: conf.OptionalString("jsonSignRoot", ""),
		OwnerName:    conf.OptionalString("ownerName", username),
		Username:     osutil.Username(),
		Prefix:       ld.MyPrefix(),
	}
	root.Stealth = conf.OptionalBool("stealth", false)
	root.statusRoot = conf.OptionalString("statusRoot", "")
	root.helpRoot = conf.OptionalString("helpRoot", "")
	if err = conf.Validate(); err != nil {
		return
	}

	if root.BlobRoot != "" {
		bs, err := ld.GetStorage(root.BlobRoot)
		if err != nil {
			return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err)
		}
		root.Storage = bs
	}

	if root.JSONSignRoot != "" {
		h, _ := ld.GetHandler(root.JSONSignRoot)
		if sigh, ok := h.(*signhandler.Handler); ok {
			root.sigh = sigh
		}
	}

	root.searchInit = func() {}
	if root.SearchRoot != "" {
		prefix := root.SearchRoot
		if t := ld.GetHandlerType(prefix); t != "search" {
			if t == "" {
				return nil, fmt.Errorf("root handler's searchRoot of %q is invalid and doesn't refer to a declared handler", prefix)
			}
			return nil, fmt.Errorf("root handler's searchRoot of %q is of type %q, not %q", prefix, t, "search")
		}
		root.searchInit = func() {
			h, err := ld.GetHandler(prefix)
			if err != nil {
				log.Fatalf("Error fetching SearchRoot at %q: %v", prefix, err)
			}
			root.searchHandler = h.(*search.Handler)
			root.searchInit = nil
		}
	}

	if pfx, _, _ := ld.FindHandlerByType("importer"); err == nil {
		root.importerRoot = pfx
	}

	return root, nil
}
Example #18
0
func uiFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	ui := &UIHandler{
		prefix:     ld.MyPrefix(),
		sourceRoot: conf.OptionalString("sourceRoot", ""),
		resizeSem: syncutil.NewSem(int64(conf.OptionalInt("maxResizeBytes",
			constants.DefaultMaxResizeMem))),
	}
	cachePrefix := conf.OptionalString("cache", "")
	scaledImageConf := conf.OptionalObject("scaledImage")
	if err = conf.Validate(); err != nil {
		return
	}

	scaledImageKV, err := newKVOrNil(scaledImageConf)
	if err != nil {
		return nil, fmt.Errorf("in UI handler's scaledImage: %v", err)
	}
	if scaledImageKV != nil && cachePrefix == "" {
		return nil, fmt.Errorf("in UI handler, can't specify scaledImage without cache")
	}
	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("UI handler's cache of %q error: %v", cachePrefix, err)
		}
		ui.Cache = bs
		ui.thumbMeta = NewThumbMeta(scaledImageKV)
	}

	if ui.sourceRoot == "" {
		ui.sourceRoot = os.Getenv("CAMLI_DEV_CAMLI_ROOT")
		if uistatic.IsAppEngine {
			if _, err = os.Stat(filepath.Join(uistatic.GaeSourceRoot,
				filepath.FromSlash("server/camlistored/ui/index.html"))); err != nil {
				hint := fmt.Sprintf("\"sourceRoot\" was not specified in the config,"+
					" and the default sourceRoot dir %v does not exist or does not contain"+
					" \"server/camlistored/ui/index.html\". devcam appengine can do that for you.",
					uistatic.GaeSourceRoot)
				log.Print(hint)
				return nil, errors.New("No sourceRoot found; UI not available.")
			}
			log.Printf("Using the default \"%v\" as the sourceRoot for AppEngine", uistatic.GaeSourceRoot)
			ui.sourceRoot = uistatic.GaeSourceRoot
		}
		if ui.sourceRoot == "" && uistatic.Files.IsEmpty() {
			ui.sourceRoot, err = osutil.GoPackagePath("camlistore.org")
			if err != nil {
				log.Printf("Warning: server not compiled with linked-in UI resources (HTML, JS, CSS), and camlistore.org not found in GOPATH.")
			} else {
				log.Printf("Using UI resources (HTML, JS, CSS) from disk, under %v", ui.sourceRoot)
			}
		}
	}
	if ui.sourceRoot != "" {
		ui.uiDir = filepath.Join(ui.sourceRoot, filepath.FromSlash("server/camlistored/ui"))
		// Ignore any fileembed files:
		Files = &fileembed.Files{
			DirFallback: filepath.Join(ui.sourceRoot, filepath.FromSlash("pkg/server")),
		}
		uistatic.Files = &fileembed.Files{
			DirFallback: ui.uiDir,
			Listable:    true,
			// In dev_appserver, allow edit-and-reload without
			// restarting. In production, though, it's faster to just
			// slurp it in.
			SlurpToMemory: uistatic.IsProdAppEngine,
		}
	}

	ui.closureHandler, err = ui.makeClosureHandler(ui.sourceRoot)
	if err != nil {
		return nil, fmt.Errorf(`Invalid "sourceRoot" value of %q: %v"`, ui.sourceRoot, err)
	}

	if ui.sourceRoot != "" {
		ui.fileReactHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "react"), "react.js")
		if err != nil {
			return nil, fmt.Errorf("Could not make react handler: %s", err)
		}
		ui.fileGlitchHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "glitch"), "npc_piggy__x1_walk_png_1354829432.png")
		if err != nil {
			return nil, fmt.Errorf("Could not make glitch handler: %s", err)
		}
		ui.fileFontawesomeHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "fontawesome"), "css/font-awesome.css")
		if err != nil {
			return nil, fmt.Errorf("Could not make fontawesome handler: %s", err)
		}
		ui.fileLessHandler, err = makeFileServer(ui.sourceRoot, filepath.Join(vendorEmbed, "less"), "less.js")
		if err != nil {
			return nil, fmt.Errorf("Could not make less handler: %s", err)
		}
	}

	rootPrefix, _, err := ld.FindHandlerByType("root")
	if err != nil {
		return nil, errors.New("No root handler configured, which is necessary for the ui handler")
	}
	if h, err := ld.GetHandler(rootPrefix); err == nil {
		ui.root = h.(*RootHandler)
		ui.root.registerUIHandler(ui)
	} else {
		return nil, errors.New("failed to find the 'root' handler")
	}

	return ui, nil
}
Example #19
0
// GenerateClientConfig retuns a client configuration which can be used to
// access a server defined by the provided low-level server configuration.
func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) {
	missingConfig := func(param string) (*Config, error) {
		return nil, fmt.Errorf("required value for %q not found", param)
	}

	if serverConfig == nil {
		return nil, errors.New("server config is a required parameter")
	}
	param := "auth"
	auth := serverConfig.OptionalString(param, "")
	if auth == "" {
		return missingConfig(param)
	}

	listen := serverConfig.OptionalString("listen", "")
	baseURL := serverConfig.OptionalString("baseURL", "")
	if listen == "" {
		listen = baseURL
	}
	if listen == "" {
		return nil, errors.New("required value for 'listen' or 'baseURL' not found")
	}

	https := serverConfig.OptionalBool("https", false)
	if !strings.HasPrefix(listen, "http://") && !strings.HasPrefix(listen, "https://") {
		if !https {
			listen = "http://" + listen
		} else {
			listen = "https://" + listen
		}
	}

	httpsCert := serverConfig.OptionalString("httpsCert", "")
	// TODO(mpl): See if we can detect that the cert is not self-signed,and in
	// that case not add it to the trustedCerts
	var trustedList []string
	if https && httpsCert != "" {
		certPEMBlock, err := wkfs.ReadFile(httpsCert)
		if err != nil {
			return nil, fmt.Errorf("could not read certificate: %v", err)
		}
		sig, err := httputil.CertFingerprint(certPEMBlock)
		if err != nil {
			return nil, fmt.Errorf("could not get fingerprints of certificate: %v", err)
		}
		trustedList = []string{sig}
	}
	param = "prefixes"
	prefixes := serverConfig.OptionalObject(param)
	if len(prefixes) == 0 {
		return missingConfig(param)
	}

	param = "/sighelper/"
	sighelper := prefixes.OptionalObject(param)
	if len(sighelper) == 0 {
		return missingConfig(param)
	}

	param = "handlerArgs"
	handlerArgs := sighelper.OptionalObject(param)
	if len(handlerArgs) == 0 {
		return missingConfig(param)
	}

	param = "keyId"
	keyId := handlerArgs.OptionalString(param, "")
	if keyId == "" {
		return missingConfig(param)
	}

	param = "secretRing"
	secretRing := handlerArgs.OptionalString(param, "")
	if secretRing == "" {
		return missingConfig(param)
	}

	return &Config{
		Servers: map[string]*Server{
			"default": {
				Server:       listen,
				Auth:         auth,
				IsDefault:    true,
				TrustedCerts: trustedList,
			},
		},
		Identity:           keyId,
		IdentitySecretRing: secretRing,
		IgnoredFiles:       []string{".DS_Store", "*~"},
	}, nil
}