func rootInTmpDir() (string, error) { user := osutil.Username() if user == "" { return "", errors.New("Could not get username from environment") } return filepath.Join(os.TempDir(), "camliroot-"+user), nil }
func makeIndex() *index.Index { dbname := "camlitest_" + osutil.Username() closeAllSessions(dbname) do(rootdb, "DROP DATABASE IF EXISTS "+dbname) do(rootdb, "CREATE DATABASE "+dbname) var err error testdb, err = sql.Open("postgres", "user=postgres password=postgres host=localhost sslmode=require dbname="+dbname) if err != nil { panic("opening test database: " + err.Error()) } for _, tableSql := range postgres.SQLCreateTables() { do(testdb, tableSql) } for _, statement := range postgres.SQLDefineReplace() { do(testdb, statement) } doQuery(testdb, fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, postgres.SchemaVersion())) s, err := postgres.NewStorage("localhost", "postgres", "postgres", dbname, "require") if err != nil { panic(err) } return index.New(s) }
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { skipOrFailIfNoMySQL(t) dbname := "camlitest_" + osutil.Username() do(rootdb, "DROP DATABASE IF EXISTS "+dbname) do(rootdb, "CREATE DATABASE "+dbname) db, err := sql.Open("mymysql", dbname+"/root/root") if err != nil { t.Fatalf("opening test database: " + err.Error()) } for _, tableSql := range mysql.SQLCreateTables() { do(db, tableSql) } do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion())) kv, err = mysql.NewKeyValue(mysql.Config{ Database: dbname, User: "******", Password: "******", }) if err != nil { t.Fatal(err) } return kv, func() { kv.Close() } }
func getUserName() (string, error) { u, err := user.Current() if err != nil { if v := osutil.Username(); v != "" { return v, nil } return "", err } return u.Name, nil }
func (c *serverCmd) setCamliRoot() error { user := osutil.Username() if user == "" { return errors.New("Could not get username from environment") } c.camliRoot = filepath.Join(os.TempDir(), "camliroot-"+user, "port"+c.port) if c.wipe { log.Printf("Wiping %v", c.camliRoot) if err := os.RemoveAll(c.camliRoot); err != nil { return fmt.Errorf("Could not wipe %v: %v", c.camliRoot, err) } } return nil }
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { username, _ := getUserName() root := &RootHandler{ BlobRoot: conf.OptionalString("blobRoot", ""), SearchRoot: conf.OptionalString("searchRoot", ""), OwnerName: conf.OptionalString("ownerName", username), Username: osutil.Username(), Prefix: ld.MyPrefix(), } root.Stealth = conf.OptionalBool("stealth", false) root.statusRoot = conf.OptionalString("statusRoot", "") if err = conf.Validate(); err != nil { return } if root.BlobRoot != "" { bs, err := ld.GetStorage(root.BlobRoot) if err != nil { return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err) } root.Storage = bs } root.searchInit = func() {} if root.SearchRoot != "" { prefix := root.SearchRoot if t := ld.GetHandlerType(prefix); t != "search" { if t == "" { return nil, fmt.Errorf("root handler's searchRoot of %q is invalid and doesn't refer to a declared handler", prefix) } return nil, fmt.Errorf("root handler's searchRoot of %q is of type %q, not %q", prefix, t, "search") } root.searchInit = func() { h, err := ld.GetHandler(prefix) if err != nil { log.Fatalf("Error fetching SearchRoot at %q: %v", prefix, err) } root.searchHandler = h.(*search.Handler) root.searchInit = nil } } if pfx, _, _ := ld.FindHandlerByType("importer"); err == nil { root.importerRoot = pfx } return root, nil }
// TestMySQLKV tests against a real MySQL instance, using a Docker container. func TestMySQLKV(t *testing.T) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupMySQLContainer(t, dbname) defer containerID.KillRemove(t) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "mysql", "host": ip + ":3306", "database": dbname, "user": dockertest.MySQLUsername, "password": dockertest.MySQLPassword, }) if err != nil { t.Fatalf("mysql.NewKeyValue = %v", err) } kvtest.TestSorted(t, kv) }
func newMongoSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupMongoContainer(t) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "mongo", "host": ip, "database": dbname, }) if err != nil { containerID.KillRemove(t) t.Fatal(err) } return kv, func() { kv.Close() containerID.KillRemove(t) } }
// TestPostgreSQLKV tests against a real PostgreSQL instance, using a Docker container. func TestPostgreSQLKV(t *testing.T) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname) defer containerID.KillRemove(t) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "postgres", "host": ip, "database": dbname, "user": dockertest.PostgresUsername, "password": dockertest.PostgresPassword, "sslmode": "disable", }) if err != nil { t.Fatalf("postgres.NewKeyValue = %v", err) } kvtest.TestSorted(t, kv) }
func makeIndex() *index.Index { dbname := "camlitest_" + osutil.Username() do(rootdb, "DROP DATABASE IF EXISTS "+dbname) do(rootdb, "CREATE DATABASE "+dbname) db, err := sql.Open("mymysql", dbname+"/root/root") if err != nil { panic("opening test database: " + err.Error()) } for _, tableSql := range mysql.SQLCreateTables() { do(db, tableSql) } do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion())) s, err := mysql.NewStorage("localhost", "root", "root", dbname) if err != nil { panic(err) } return index.New(s) }
// dbName returns which database to use for the provided user ("of"). // The user should be a key as described in pkg/types/serverconfig/config.go's // description of DBNames: "index", "queue-sync-to-index", etc. func (b *lowBuilder) dbName(of string) string { if v, ok := b.high.DBNames[of]; ok && v != "" { return v } if of == "index" { if b.high.DBName != "" { return b.high.DBName } username := osutil.Username() if username == "" { envVar := "USER" if runtime.GOOS == "windows" { envVar += "NAME" } return "camlistore_index" } return "camli" + username } return "" }
func newMySQLSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupMySQLContainer(t, dbname) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "mysql", "host": ip + ":3306", "database": dbname, "user": dockertest.MySQLUsername, "password": dockertest.MySQLPassword, }) if err != nil { containerID.KillRemove(t) t.Fatal(err) } return kv, func() { kv.Close() containerID.KillRemove(t) } }
// TestMySQLKV tests against a real MySQL instance, using a Docker container. func TestMySQLKV(t *testing.T) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupMySQLContainer(t, dbname) defer containerID.KillRemove(t) // TODO(mpl): add test for serverVersion once we host the docker image ourselves // (and hence have the control over the version). kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "mysql", "host": ip + ":3306", "database": dbname, "user": dockertest.MySQLUsername, "password": dockertest.MySQLPassword, }) if err != nil { t.Fatalf("mysql.NewKeyValue = %v", err) } kvtest.TestSorted(t, kv) }
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { skipOrFailIfNoPostgreSQL(t) dbname := "camlitest_" + osutil.Username() if err := closeAllSessions(postgres.Config{ User: "******", Password: "******", SSLMode: "require", Database: dbname, Host: "localhost", }); err != nil { t.Fatalf("Could not close all old sessions to %q: %v", dbname, err) } do(rootdb, "DROP DATABASE IF EXISTS "+dbname) do(rootdb, "CREATE DATABASE "+dbname+" LC_COLLATE = 'C' TEMPLATE = template0") testdb, err := sql.Open("postgres", "user=postgres password=postgres host=localhost sslmode=require dbname="+dbname) if err != nil { t.Fatalf("opening test database: " + err.Error()) } for _, tableSql := range postgres.SQLCreateTables() { do(testdb, tableSql) } for _, statement := range postgres.SQLDefineReplace() { do(testdb, statement) } doQuery(testdb, fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, postgres.SchemaVersion())) kv, err = postgres.NewKeyValue(postgres.Config{ Host: "localhost", Database: dbname, User: "******", Password: "******", SSLMode: "require", }) if err != nil { t.Fatal(err) } return kv, func() { kv.Close() } }
func newPostgresSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "postgres", "host": ip, "database": dbname, "user": dockertest.PostgresUsername, "password": dockertest.PostgresPassword, "sslmode": "disable", }) if err != nil { containerID.KillRemove(t) t.Fatal(err) } return kv, func() { kv.Close() containerID.KillRemove(t) } }
func TestRollback(t *testing.T) { dbname := "camlitest_" + osutil.Username() containerID, ip := dockertest.SetupMySQLContainer(t, dbname) defer containerID.KillRemove(t) kv, err := sorted.NewKeyValue(jsonconfig.Obj{ "type": "mysql", "host": ip + ":3306", "database": dbname, "user": dockertest.MySQLUsername, "password": dockertest.MySQLPassword, }) if err != nil { t.Fatalf("mysql.NewKeyValue = %v", err) } tooLargeAKey := make([]byte, sorted.MaxKeySize+10) for i := range tooLargeAKey { tooLargeAKey[i] = 'L' } nbConnections := 2 tick := time.AfterFunc(5*time.Second, func() { // We have to force close the connection, otherwise the connection hogging does not even // let us exit the func with t.Fatal (How? why?) kv.(*keyValue).DB.Close() t.Fatal("Test failed because SQL connections blocked by unrolled transactions") }) kv.(*keyValue).DB.SetMaxOpenConns(nbConnections) for i := 0; i < nbConnections+1; i++ { b := kv.BeginBatch() // Making the transaction fail, to force a rollback // -> this whole test fails before we introduce the rollback in CommitBatch. b.Set(string(tooLargeAKey), "whatever") if err := kv.CommitBatch(b); err == nil { t.Fatal("wanted failed commit because too large a key") } } tick.Stop() }
func (c *serverCmd) setEnvVars() error { c.env.SetCamdevVars(false) setenv := func(k, v string) { c.env.Set(k, v) } if c.slow { setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle)) setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency)) } if c.debug { setenv("CAMLI_HTTP_DEBUG", "1") } user := osutil.Username() if user == "" { return errors.New("Could not get username from environment") } setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false") if c.fullIndexSync { setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "true") } setenv("CAMLI_DBNAME", "devcamli"+user) setenv("CAMLI_MYSQL_ENABLED", "false") setenv("CAMLI_MONGO_ENABLED", "false") setenv("CAMLI_POSTGRES_ENABLED", "false") setenv("CAMLI_SQLITE_ENABLED", "false") setenv("CAMLI_KVINDEX_ENABLED", "false") setenv("CAMLI_MEMINDEX_ENABLED", "false") setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish)) setenv("CAMLI_HELLO_ENABLED", strconv.FormatBool(c.hello)) switch { case c.memory: setenv("CAMLI_MEMINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-memory/") case c.mongo: setenv("CAMLI_MONGO_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mongo/") case c.postgres: setenv("CAMLI_POSTGRES_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-postgres/") case c.mysql: setenv("CAMLI_MYSQL_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mysql/") case c.sqlite: setenv("CAMLI_SQLITE_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-sqlite/") if c.root == "" { panic("no root set") } setenv("CAMLI_DBNAME", filepath.Join(c.root, "sqliteindex.db")) default: setenv("CAMLI_KVINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-kv/") if c.root == "" { panic("no root set") } setenv("CAMLI_DBNAME", filepath.Join(c.root, "kvindex.db")) } base := "http://localhost:" + c.port c.listen = "127.0.0.1:" + c.port if c.all { c.listen = "0.0.0.0:" + c.port if c.hostname == "" { hostname, err := os.Hostname() if err != nil { return fmt.Errorf("Could not get system hostname: %v", err) } base = "http://" + hostname + ":" + c.port } else { base = "http://" + c.hostname + ":" + c.port } } setenv("CAMLI_TLS", "false") if c.tls { base = strings.Replace(base, "http://", "https://", 1) setenv("CAMLI_TLS", "true") } setenv("CAMLI_BASEURL", base) setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot) setenv("CAMLI_AUTH", "devauth:pass3179") fullSuffix := func(name string) string { return filepath.Join(c.root, name) } suffixes := map[string]string{ "CAMLI_ROOT": fullSuffix("bs"), "CAMLI_ROOT_SHARD1": fullSuffix("s1"), "CAMLI_ROOT_SHARD2": fullSuffix("s2"), "CAMLI_ROOT_REPLICA1": fullSuffix("r1"), "CAMLI_ROOT_REPLICA2": fullSuffix("r2"), "CAMLI_ROOT_REPLICA3": fullSuffix("r3"), "CAMLI_ROOT_CACHE": fullSuffix("cache"), "CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"), "CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"), } for k, v := range suffixes { c.makeSuffixdir(v) setenv(k, v) } c.makeSuffixdir(filepath.Join(fullSuffix("bs"), "packed")) c.makeSuffixdir(filepath.Join(fullSuffix("bs"), "loose")) setenv("CAMLI_PORT", c.port) if c.flickrAPIKey != "" { setenv("CAMLI_FLICKR_ENABLED", "true") setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey) } if c.foursquareAPIKey != "" { setenv("CAMLI_FOURSQUARE_ENABLED", "true") setenv("CAMLI_FOURSQUARE_API_KEY", c.foursquareAPIKey) } if c.picasaAPIKey != "" { setenv("CAMLI_PICASA_ENABLED", "true") setenv("CAMLI_PICASA_API_KEY", c.picasaAPIKey) } if c.twitterAPIKey != "" { setenv("CAMLI_TWITTER_ENABLED", "true") setenv("CAMLI_TWITTER_API_KEY", c.twitterAPIKey) } setenv("CAMLI_CONFIG_DIR", "config") setenv("CAMLI_CACHE_DIR", filepath.Join(c.root, "cache")) setenv("CAMLI_APP_BINDIR", "bin") return nil }
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { checkType := func(key string, htype string) { v := conf.OptionalString(key, "") if v == "" { return } ct := ld.GetHandlerType(v) if ct == "" { err = fmt.Errorf("root handler's %q references non-existant %q", key, v) } else if ct != htype { err = fmt.Errorf("root handler's %q references %q of type %q; expected type %q", key, v, ct, htype) } } checkType("searchRoot", "search") checkType("jsonSignRoot", "jsonsign") if err != nil { return } username, _ := getUserName() root := &RootHandler{ BlobRoot: conf.OptionalString("blobRoot", ""), SearchRoot: conf.OptionalString("searchRoot", ""), JSONSignRoot: conf.OptionalString("jsonSignRoot", ""), OwnerName: conf.OptionalString("ownerName", username), Username: osutil.Username(), Prefix: ld.MyPrefix(), } root.Stealth = conf.OptionalBool("stealth", false) root.statusRoot = conf.OptionalString("statusRoot", "") root.helpRoot = conf.OptionalString("helpRoot", "") if err = conf.Validate(); err != nil { return } if root.BlobRoot != "" { bs, err := ld.GetStorage(root.BlobRoot) if err != nil { return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err) } root.Storage = bs } if root.JSONSignRoot != "" { h, _ := ld.GetHandler(root.JSONSignRoot) if sigh, ok := h.(*signhandler.Handler); ok { root.sigh = sigh } } root.searchInit = func() {} if root.SearchRoot != "" { prefix := root.SearchRoot if t := ld.GetHandlerType(prefix); t != "search" { if t == "" { return nil, fmt.Errorf("root handler's searchRoot of %q is invalid and doesn't refer to a declared handler", prefix) } return nil, fmt.Errorf("root handler's searchRoot of %q is of type %q, not %q", prefix, t, "search") } root.searchInit = func() { h, err := ld.GetHandler(prefix) if err != nil { log.Fatalf("Error fetching SearchRoot at %q: %v", prefix, err) } root.searchHandler = h.(*search.Handler) root.searchInit = nil } } if pfx, _, _ := ld.FindHandlerByType("importer"); err == nil { root.importerRoot = pfx } return root, nil }
func (c *serverCmd) setEnvVars() error { setenv := func(k, v string) { c.env.Set(k, v) } if c.slow { setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle)) setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency)) } if c.debug { setenv("CAMLI_HTTP_DEBUG", "1") } user := osutil.Username() if user == "" { return errors.New("Could not get username from environment") } setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false") // TODO: option to make this true setenv("CAMLI_DBNAME", "devcamli"+user) setenv("CAMLI_MYSQL_ENABLED", "false") setenv("CAMLI_MONGO_ENABLED", "false") setenv("CAMLI_POSTGRES_ENABLED", "false") setenv("CAMLI_SQLITE_ENABLED", "false") setenv("CAMLI_KVINDEX_ENABLED", "false") switch { case c.mongo: setenv("CAMLI_MONGO_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mongo/") case c.postgres: setenv("CAMLI_POSTGRES_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-postgres/") case c.mysql: setenv("CAMLI_MYSQL_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mysql/") case c.sqlite: setenv("CAMLI_SQLITE_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-sqlite/") if c.camliRoot == "" { panic("no camliRoot set") } setenv("CAMLI_DBNAME", filepath.Join(c.camliRoot, "sqliteindex.db")) default: setenv("CAMLI_KVINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-kv/") if c.camliRoot == "" { panic("no camliRoot set") } setenv("CAMLI_DBNAME", filepath.Join(c.camliRoot, "kvindex.db")) } base := "http://localhost:" + c.port c.listen = "127.0.0.1:" + c.port if c.all { c.listen = "0.0.0.0:" + c.port if c.hostname == "" { hostname, err := os.Hostname() if err != nil { return fmt.Errorf("Could not get system hostname: %v", err) } base = "http://" + hostname + ":" + c.port } else { base = "http://" + c.hostname + ":" + c.port } } setenv("CAMLI_TLS", "false") if c.tls { base = strings.Replace(base, "http://", "https://", 1) setenv("CAMLI_TLS", "true") } setenv("CAMLI_BASEURL", base) setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot) setenv("CAMLI_AUTH", "devauth:pass3179") fullSuffix := func(name string) string { return filepath.Join(c.camliRoot, name) } suffixes := map[string]string{ "CAMLI_ROOT": fullSuffix("bs"), "CAMLI_ROOT_SHARD1": fullSuffix("s1"), "CAMLI_ROOT_SHARD2": fullSuffix("s2"), "CAMLI_ROOT_REPLICA1": fullSuffix("r1"), "CAMLI_ROOT_REPLICA2": fullSuffix("r2"), "CAMLI_ROOT_REPLICA3": fullSuffix("r3"), "CAMLI_ROOT_CACHE": fullSuffix("cache"), "CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"), "CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"), } for k, v := range suffixes { c.makeSuffixdir(v) setenv(k, v) } setenv("CAMLI_PORT", c.port) setenv("CAMLI_SECRET_RING", filepath.Join(camliSrcRoot, filepath.FromSlash(defaultSecring))) setenv("CAMLI_KEYID", defaultKeyID) if c.flickrAPIKey != "" { setenv("CAMLI_FLICKR_ENABLED", "true") setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey) } setenv("CAMLI_CONFIG_DIR", "config") return nil }
// genLowLevelConfig returns a low-level config from a high-level config. func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) { obj := jsonconfig.Obj{} if conf.HTTPS { if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") { return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)") } if conf.HTTPSCert != "" { obj["httpsCert"] = conf.HTTPSCert obj["httpsKey"] = conf.HTTPSKey } else { obj["httpsCert"] = osutil.DefaultTLSCert() obj["httpsKey"] = osutil.DefaultTLSKey() } } if conf.BaseURL != "" { u, err := url.Parse(conf.BaseURL) if err != nil { return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", conf.BaseURL, err) } if u.Path != "" && u.Path != "/" { return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.") } u.Path = "" obj["baseURL"] = u.String() } if conf.Listen != "" { obj["listen"] = conf.Listen } obj["https"] = conf.HTTPS obj["auth"] = conf.Auth username := "" if conf.DBName == "" { username = osutil.Username() if username == "" { return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname") } conf.DBName = "camli" + username } var indexerPath string numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile) runIndex := conf.RunIndex.Get() switch { case runIndex && numIndexers == 0: return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).") case runIndex && numIndexers != 1: return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).") case !runIndex && numIndexers != 0: return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.") case conf.MySQL != "": indexerPath = "/index-mysql/" case conf.PostgreSQL != "": indexerPath = "/index-postgres/" case conf.Mongo != "": indexerPath = "/index-mongo/" case conf.SQLite != "": indexerPath = "/index-sqlite/" case conf.KVFile != "": indexerPath = "/index-kv/" } entity, err := jsonsign.EntityFromSecring(conf.Identity, conf.IdentitySecretRing) if err != nil { return nil, err } armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity) if err != nil { return nil, err } nolocaldisk := conf.BlobPath == "" if nolocaldisk { if conf.S3 == "" && conf.GoogleCloudStorage == "" { return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.") } if conf.S3 != "" && conf.GoogleCloudStorage != "" { return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.") } } if conf.ShareHandler && conf.ShareHandlerPath == "" { conf.ShareHandlerPath = "/share/" } prefixesParams := &configPrefixesParams{ secretRing: conf.IdentitySecretRing, keyId: conf.Identity, indexerPath: indexerPath, blobPath: conf.BlobPath, packBlobs: conf.PackBlobs, searchOwner: blob.SHA1FromString(armoredPublicKey), shareHandlerPath: conf.ShareHandlerPath, flickr: conf.Flickr, memoryIndex: conf.MemoryIndex.Get(), } prefixes := genLowLevelPrefixes(prefixesParams, conf.OwnerName) var cacheDir string if nolocaldisk { // Whether camlistored is run from EC2 or not, we use // a temp dir as the cache when primary storage is S3. // TODO(mpl): s3CacheBucket // See http://code.google.com/p/camlistore/issues/detail?id=85 cacheDir = filepath.Join(tempDir(), "camli-cache") } else { cacheDir = filepath.Join(conf.BlobPath, "cache") } if !noMkdir { if err := os.MkdirAll(cacheDir, 0700); err != nil { return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err) } } published := []interface{}{} if len(conf.Publish) > 0 { if !runIndex { return nil, fmt.Errorf("publishing requires an index") } published, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot) if err != nil { return nil, fmt.Errorf("Could not generate config for published: %v", err) } } if runIndex { addUIConfig(prefixesParams, prefixes, "/ui/", published, conf.SourceRoot) } if conf.MySQL != "" { addMySQLConfig(prefixes, conf.DBName, conf.MySQL) } if conf.PostgreSQL != "" { addPostgresConfig(prefixes, conf.DBName, conf.PostgreSQL) } if conf.Mongo != "" { addMongoConfig(prefixes, conf.DBName, conf.Mongo) } if conf.SQLite != "" { addSQLiteConfig(prefixes, conf.SQLite) } if conf.KVFile != "" { addKVConfig(prefixes, conf.KVFile) } if conf.S3 != "" { if err := addS3Config(prefixesParams, prefixes, conf.S3); err != nil { return nil, err } } if conf.GoogleDrive != "" { if err := addGoogleDriveConfig(prefixes, conf.GoogleDrive); err != nil { return nil, err } } if conf.GoogleCloudStorage != "" { if err := addGoogleCloudStorageConfig(prefixes, conf.GoogleCloudStorage); err != nil { return nil, err } } obj["prefixes"] = (map[string]interface{})(prefixes) lowLevelConf = &Config{ Obj: obj, } return lowLevelConf, nil }
// genLowLevelConfig returns a low-level config from a high-level config. func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { var ( baseURL = conf.OptionalString("baseURL", "") listen = conf.OptionalString("listen", "") auth = conf.RequiredString("auth") keyId = conf.RequiredString("identity") secretRing = conf.RequiredString("identitySecretRing") tlsOn = conf.OptionalBool("https", false) tlsCert = conf.OptionalString("HTTPSCertFile", "") tlsKey = conf.OptionalString("HTTPSKeyFile", "") // Blob storage options blobPath = conf.OptionalString("blobPath", "") packBlobs = conf.OptionalBool("packBlobs", false) // use diskpacked instead of the default filestorage s3 = conf.OptionalString("s3", "") // "access_key_id:secret_access_key:bucket[:hostname]" googlecloudstorage = conf.OptionalString("googlecloudstorage", "") // "clientId:clientSecret:refreshToken:bucket" googledrive = conf.OptionalString("googledrive", "") // "clientId:clientSecret:refreshToken:parentId" swift = conf.OptionalString("swift", "") // "tenant:secret:container:auth_url" // Enable the share handler. If true, and shareHandlerPath is empty, // then shareHandlerPath defaults to "/share/". shareHandler = conf.OptionalBool("shareHandler", false) // URL prefix for the share handler. If set, overrides shareHandler. shareHandlerPath = conf.OptionalString("shareHandlerPath", "") // Index options memoryIndex = conf.OptionalBool("memoryIndex", true) // copy disk-based index to memory on start-up runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc. dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo mysql = conf.OptionalString("mysql", "") postgres = conf.OptionalString("postgres", "") mongo = conf.OptionalString("mongo", "") sqliteFile = conf.OptionalString("sqlite", "") kvFile = conf.OptionalString("kvIndexFile", "") // Importer options flickr = conf.OptionalString("flickr", "") _ = conf.OptionalList("replicateTo") publish = conf.OptionalObject("publish") // alternative source tree, to override the embedded ui and/or closure resources. // If non empty, the ui files will be expected at // sourceRoot + "/server/camlistored/ui" and the closure library at // sourceRoot + "/third_party/closure/lib" // Also used by the publish handler. sourceRoot = conf.OptionalString("sourceRoot", "") ownerName = conf.OptionalString("ownerName", "") ) if err := conf.Validate(); err != nil { return nil, err } obj := jsonconfig.Obj{} if tlsOn { if (tlsCert != "") != (tlsKey != "") { return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)") } if tlsCert != "" { obj["TLSCertFile"] = tlsCert obj["TLSKeyFile"] = tlsKey } else { obj["TLSCertFile"] = osutil.DefaultTLSCert() obj["TLSKeyFile"] = osutil.DefaultTLSKey() } } if baseURL != "" { u, err := url.Parse(baseURL) if err != nil { return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", baseURL, err) } if u.Path != "" && u.Path != "/" { return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.") } u.Path = "" obj["baseURL"] = u.String() } if listen != "" { obj["listen"] = listen } obj["https"] = tlsOn obj["auth"] = auth username := "" if dbname == "" { username = osutil.Username() if username == "" { return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname") } dbname = "camli" + username } var indexerPath string numIndexers := numSet(mongo, mysql, postgres, sqliteFile, kvFile) switch { case runIndex && numIndexers == 0: return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).") case runIndex && numIndexers != 1: return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).") case !runIndex && numIndexers != 0: return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.") case mysql != "": indexerPath = "/index-mysql/" case postgres != "": indexerPath = "/index-postgres/" case mongo != "": indexerPath = "/index-mongo/" case sqliteFile != "": indexerPath = "/index-sqlite/" case kvFile != "": indexerPath = "/index-kv/" } entity, err := jsonsign.EntityFromSecring(keyId, secretRing) if err != nil { return nil, err } armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity) if err != nil { return nil, err } nolocaldisk := blobPath == "" if nolocaldisk { if s3 == "" && googlecloudstorage == "" { return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.") } if s3 != "" && googlecloudstorage != "" { return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.") } } if shareHandler && shareHandlerPath == "" { shareHandlerPath = "/share/" } prefixesParams := &configPrefixesParams{ secretRing: secretRing, keyId: keyId, indexerPath: indexerPath, blobPath: blobPath, packBlobs: packBlobs, searchOwner: blob.SHA1FromString(armoredPublicKey), shareHandlerPath: shareHandlerPath, flickr: flickr, memoryIndex: memoryIndex, } prefixes := genLowLevelPrefixes(prefixesParams, ownerName) var cacheDir string if nolocaldisk { // Whether camlistored is run from EC2 or not, we use // a temp dir as the cache when primary storage is S3. // TODO(mpl): s3CacheBucket // See http://code.google.com/p/camlistore/issues/detail?id=85 cacheDir = filepath.Join(tempDir(), "camli-cache") } else { cacheDir = filepath.Join(blobPath, "cache") } if !noMkdir { if err := os.MkdirAll(cacheDir, 0700); err != nil { return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err) } } published := []interface{}{} if len(publish) > 0 { if !runIndex { return nil, fmt.Errorf("publishing requires an index") } published, err = addPublishedConfig(prefixes, publish, sourceRoot) if err != nil { return nil, fmt.Errorf("Could not generate config for published: %v", err) } } if runIndex { addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot) } if mysql != "" { addMySQLConfig(prefixes, dbname, mysql) } if postgres != "" { addPostgresConfig(prefixes, dbname, postgres) } if mongo != "" { addMongoConfig(prefixes, dbname, mongo) } if sqliteFile != "" { addSQLiteConfig(prefixes, sqliteFile) } if kvFile != "" { addKVConfig(prefixes, kvFile) } if s3 != "" { if err := addS3Config(prefixesParams, prefixes, s3); err != nil { return nil, err } } if googledrive != "" { if err := addGoogleDriveConfig(prefixes, googledrive); err != nil { return nil, err } } if googlecloudstorage != "" { if err := addGoogleCloudStorageConfig(prefixes, googlecloudstorage); err != nil { return nil, err } } if swift != "" { if err := addSwiftConfig(prefixesParams, prefixes, swift); err != nil { return nil, err } } obj["prefixes"] = (map[string]interface{})(prefixes) lowLevelConf = &Config{ Obj: obj, configPath: conf.configPath, } return lowLevelConf, nil }
func (c *serverCmd) setEnvVars() error { if c.slow { setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle)) setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency)) } if c.debug { setenv("CAMLI_HTTP_DEBUG", "1") } user := osutil.Username() if user == "" { return errors.New("Could not get username from environment") } setenv("CAMLI_DBNAME", "devcamli"+user) setenv("CAMLI_MYSQL_ENABLED", "false") setenv("CAMLI_MONGO_ENABLED", "false") setenv("CAMLI_POSTGRES_ENABLED", "false") setenv("CAMLI_KVINDEX_ENABLED", "false") setenv("CAMLI_MEMINDEX_ENABLED", "false") switch { case c.mongo: setenv("CAMLI_MONGO_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mongo/") case c.postgres: setenv("CAMLI_POSTGRES_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-postgres/") case c.mysql: setenv("CAMLI_MYSQL_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mysql/") case c.memindex: setenv("CAMLI_MEMINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-mem/") default: setenv("CAMLI_KVINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-kv/") if c.camliRoot == "" { panic("no camliRoot set") } setenv("CAMLI_KVINDEX_FILE", filepath.Join(c.camliRoot, "kvindex.db")) } base := "http://localhost:" + c.port c.listen = "127.0.0.1:" + c.port if c.all { c.listen = "0.0.0.0:" + c.port if c.hostname == "" { hostname, err := os.Hostname() if err != nil { return fmt.Errorf("Could not get system hostname: %v", err) } base = "http://" + hostname + ":" + c.port } else { base = "http://" + c.hostname + ":" + c.port } } setenv("CAMLI_TLS", "false") if c.tls { base = strings.Replace(base, "http://", "https://", 1) setenv("CAMLI_TLS", "true") } setenv("CAMLI_BASEURL", base) setenv("CAMLI_DEV_CAMLI_ROOT", c.camliSrcRoot) setenv("CAMLI_AUTH", "devauth:pass"+c.port) fullSuffix := func(name string) string { return filepath.Join(c.camliRoot, name) } suffixes := map[string]string{ "CAMLI_ROOT": fullSuffix("bs"), "CAMLI_ROOT_SHARD1": fullSuffix("s1"), "CAMLI_ROOT_SHARD2": fullSuffix("s2"), "CAMLI_ROOT_REPLICA1": fullSuffix("r1"), "CAMLI_ROOT_REPLICA2": fullSuffix("r2"), "CAMLI_ROOT_REPLICA3": fullSuffix("r3"), "CAMLI_ROOT_CACHE": fullSuffix("cache"), "CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"), "CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"), } for k, v := range suffixes { c.makeSuffixdir(v) setenv(k, v) } setenv("CAMLI_PORT", c.port) setenv("CAMLI_SECRET_RING", filepath.Join(c.camliSrcRoot, filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg"))) return nil }