func rootInTmpDir() (string, error) {
user := osutil.Username()
if user == "" {
return "", errors.New("Could not get username from environment")
}
return filepath.Join(os.TempDir(), "camliroot-"+user), nil
}
func makeIndex() *index.Index {
dbname := "camlitest_" + osutil.Username()
closeAllSessions(dbname)
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname)
var err error
testdb, err = sql.Open("postgres", "user=postgres password=postgres host=localhost sslmode=require dbname="+dbname)
if err != nil {
panic("opening test database: " + err.Error())
}
for _, tableSql := range postgres.SQLCreateTables() {
do(testdb, tableSql)
}
for _, statement := range postgres.SQLDefineReplace() {
do(testdb, statement)
}
doQuery(testdb, fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, postgres.SchemaVersion()))
s, err := postgres.NewStorage("localhost", "postgres", "postgres", dbname, "require")
if err != nil {
panic(err)
}
return index.New(s)
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
skipOrFailIfNoMySQL(t)
dbname := "camlitest_" + osutil.Username()
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname)
db, err := sql.Open("mymysql", dbname+"/root/root")
if err != nil {
t.Fatalf("opening test database: " + err.Error())
}
for _, tableSql := range mysql.SQLCreateTables() {
do(db, tableSql)
}
do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion()))
kv, err = mysql.NewKeyValue(mysql.Config{
Database: dbname,
User: "******",
Password: "******",
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
}
}
func getUserName() (string, error) {
u, err := user.Current()
if err != nil {
if v := osutil.Username(); v != "" {
return v, nil
}
return "", err
}
return u.Name, nil
}
func (c *serverCmd) setCamliRoot() error {
user := osutil.Username()
if user == "" {
return errors.New("Could not get username from environment")
}
c.camliRoot = filepath.Join(os.TempDir(), "camliroot-"+user, "port"+c.port)
if c.wipe {
log.Printf("Wiping %v", c.camliRoot)
if err := os.RemoveAll(c.camliRoot); err != nil {
return fmt.Errorf("Could not wipe %v: %v", c.camliRoot, err)
}
}
return nil
}
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
username, _ := getUserName()
root := &RootHandler{
BlobRoot: conf.OptionalString("blobRoot", ""),
SearchRoot: conf.OptionalString("searchRoot", ""),
OwnerName: conf.OptionalString("ownerName", username),
Username: osutil.Username(),
Prefix: ld.MyPrefix(),
}
root.Stealth = conf.OptionalBool("stealth", false)
root.statusRoot = conf.OptionalString("statusRoot", "")
if err = conf.Validate(); err != nil {
return
}
if root.BlobRoot != "" {
bs, err := ld.GetStorage(root.BlobRoot)
if err != nil {
return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err)
}
root.Storage = bs
}
root.searchInit = func() {}
if root.SearchRoot != "" {
prefix := root.SearchRoot
if t := ld.GetHandlerType(prefix); t != "search" {
if t == "" {
return nil, fmt.Errorf("root handler's searchRoot of %q is invalid and doesn't refer to a declared handler", prefix)
}
return nil, fmt.Errorf("root handler's searchRoot of %q is of type %q, not %q", prefix, t, "search")
}
root.searchInit = func() {
h, err := ld.GetHandler(prefix)
if err != nil {
log.Fatalf("Error fetching SearchRoot at %q: %v", prefix, err)
}
root.searchHandler = h.(*search.Handler)
root.searchInit = nil
}
}
if pfx, _, _ := ld.FindHandlerByType("importer"); err == nil {
root.importerRoot = pfx
}
return root, nil
}
// TestMySQLKV tests against a real MySQL instance, using a Docker container.
func TestMySQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func newMongoSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMongoContainer(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mongo",
"host": ip,
"database": dbname,
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
// TestPostgreSQLKV tests against a real PostgreSQL instance, using a Docker container.
func TestPostgreSQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "postgres",
"host": ip,
"database": dbname,
"user": dockertest.PostgresUsername,
"password": dockertest.PostgresPassword,
"sslmode": "disable",
})
if err != nil {
t.Fatalf("postgres.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func makeIndex() *index.Index {
dbname := "camlitest_" + osutil.Username()
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname)
db, err := sql.Open("mymysql", dbname+"/root/root")
if err != nil {
panic("opening test database: " + err.Error())
}
for _, tableSql := range mysql.SQLCreateTables() {
do(db, tableSql)
}
do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, mysql.SchemaVersion()))
s, err := mysql.NewStorage("localhost", "root", "root", dbname)
if err != nil {
panic(err)
}
return index.New(s)
}
// dbName returns which database to use for the provided user ("of").
// The user should be a key as described in pkg/types/serverconfig/config.go's
// description of DBNames: "index", "queue-sync-to-index", etc.
func (b *lowBuilder) dbName(of string) string {
if v, ok := b.high.DBNames[of]; ok && v != "" {
return v
}
if of == "index" {
if b.high.DBName != "" {
return b.high.DBName
}
username := osutil.Username()
if username == "" {
envVar := "USER"
if runtime.GOOS == "windows" {
envVar += "NAME"
}
return "camlistore_index"
}
return "camli" + username
}
return ""
}
func newMySQLSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
// TestMySQLKV tests against a real MySQL instance, using a Docker container.
func TestMySQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
// TODO(mpl): add test for serverVersion once we host the docker image ourselves
// (and hence have the control over the version).
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
skipOrFailIfNoPostgreSQL(t)
dbname := "camlitest_" + osutil.Username()
if err := closeAllSessions(postgres.Config{
User: "******",
Password: "******",
SSLMode: "require",
Database: dbname,
Host: "localhost",
}); err != nil {
t.Fatalf("Could not close all old sessions to %q: %v", dbname, err)
}
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname+" LC_COLLATE = 'C' TEMPLATE = template0")
testdb, err := sql.Open("postgres", "user=postgres password=postgres host=localhost sslmode=require dbname="+dbname)
if err != nil {
t.Fatalf("opening test database: " + err.Error())
}
for _, tableSql := range postgres.SQLCreateTables() {
do(testdb, tableSql)
}
for _, statement := range postgres.SQLDefineReplace() {
do(testdb, statement)
}
doQuery(testdb, fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, postgres.SchemaVersion()))
kv, err = postgres.NewKeyValue(postgres.Config{
Host: "localhost",
Database: dbname,
User: "******",
Password: "******",
SSLMode: "require",
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
}
}
func newPostgresSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "postgres",
"host": ip,
"database": dbname,
"user": dockertest.PostgresUsername,
"password": dockertest.PostgresPassword,
"sslmode": "disable",
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
func TestRollback(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
tooLargeAKey := make([]byte, sorted.MaxKeySize+10)
for i := range tooLargeAKey {
tooLargeAKey[i] = 'L'
}
nbConnections := 2
tick := time.AfterFunc(5*time.Second, func() {
// We have to force close the connection, otherwise the connection hogging does not even
// let us exit the func with t.Fatal (How? why?)
kv.(*keyValue).DB.Close()
t.Fatal("Test failed because SQL connections blocked by unrolled transactions")
})
kv.(*keyValue).DB.SetMaxOpenConns(nbConnections)
for i := 0; i < nbConnections+1; i++ {
b := kv.BeginBatch()
// Making the transaction fail, to force a rollback
// -> this whole test fails before we introduce the rollback in CommitBatch.
b.Set(string(tooLargeAKey), "whatever")
if err := kv.CommitBatch(b); err == nil {
t.Fatal("wanted failed commit because too large a key")
}
}
tick.Stop()
}
func (c *serverCmd) setEnvVars() error {
c.env.SetCamdevVars(false)
setenv := func(k, v string) {
c.env.Set(k, v)
}
if c.slow {
setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle))
setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency))
}
if c.debug {
setenv("CAMLI_HTTP_DEBUG", "1")
}
user := osutil.Username()
if user == "" {
return errors.New("Could not get username from environment")
}
setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false")
if c.fullIndexSync {
setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "true")
}
setenv("CAMLI_DBNAME", "devcamli"+user)
setenv("CAMLI_MYSQL_ENABLED", "false")
setenv("CAMLI_MONGO_ENABLED", "false")
setenv("CAMLI_POSTGRES_ENABLED", "false")
setenv("CAMLI_SQLITE_ENABLED", "false")
setenv("CAMLI_KVINDEX_ENABLED", "false")
setenv("CAMLI_MEMINDEX_ENABLED", "false")
setenv("CAMLI_PUBLISH_ENABLED", strconv.FormatBool(c.publish))
setenv("CAMLI_HELLO_ENABLED", strconv.FormatBool(c.hello))
switch {
case c.memory:
setenv("CAMLI_MEMINDEX_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-memory/")
case c.mongo:
setenv("CAMLI_MONGO_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mongo/")
case c.postgres:
setenv("CAMLI_POSTGRES_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-postgres/")
case c.mysql:
setenv("CAMLI_MYSQL_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mysql/")
case c.sqlite:
setenv("CAMLI_SQLITE_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-sqlite/")
if c.root == "" {
panic("no root set")
}
setenv("CAMLI_DBNAME", filepath.Join(c.root, "sqliteindex.db"))
default:
setenv("CAMLI_KVINDEX_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-kv/")
if c.root == "" {
panic("no root set")
}
setenv("CAMLI_DBNAME", filepath.Join(c.root, "kvindex.db"))
}
base := "http://localhost:" + c.port
c.listen = "127.0.0.1:" + c.port
if c.all {
c.listen = "0.0.0.0:" + c.port
if c.hostname == "" {
hostname, err := os.Hostname()
if err != nil {
return fmt.Errorf("Could not get system hostname: %v", err)
}
base = "http://" + hostname + ":" + c.port
} else {
base = "http://" + c.hostname + ":" + c.port
}
}
setenv("CAMLI_TLS", "false")
if c.tls {
base = strings.Replace(base, "http://", "https://", 1)
setenv("CAMLI_TLS", "true")
}
setenv("CAMLI_BASEURL", base)
setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot)
setenv("CAMLI_AUTH", "devauth:pass3179")
fullSuffix := func(name string) string {
return filepath.Join(c.root, name)
}
suffixes := map[string]string{
"CAMLI_ROOT": fullSuffix("bs"),
"CAMLI_ROOT_SHARD1": fullSuffix("s1"),
"CAMLI_ROOT_SHARD2": fullSuffix("s2"),
"CAMLI_ROOT_REPLICA1": fullSuffix("r1"),
"CAMLI_ROOT_REPLICA2": fullSuffix("r2"),
"CAMLI_ROOT_REPLICA3": fullSuffix("r3"),
"CAMLI_ROOT_CACHE": fullSuffix("cache"),
"CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"),
"CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"),
}
for k, v := range suffixes {
c.makeSuffixdir(v)
setenv(k, v)
}
c.makeSuffixdir(filepath.Join(fullSuffix("bs"), "packed"))
c.makeSuffixdir(filepath.Join(fullSuffix("bs"), "loose"))
setenv("CAMLI_PORT", c.port)
if c.flickrAPIKey != "" {
setenv("CAMLI_FLICKR_ENABLED", "true")
setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey)
}
if c.foursquareAPIKey != "" {
setenv("CAMLI_FOURSQUARE_ENABLED", "true")
setenv("CAMLI_FOURSQUARE_API_KEY", c.foursquareAPIKey)
}
if c.picasaAPIKey != "" {
setenv("CAMLI_PICASA_ENABLED", "true")
setenv("CAMLI_PICASA_API_KEY", c.picasaAPIKey)
}
if c.twitterAPIKey != "" {
setenv("CAMLI_TWITTER_ENABLED", "true")
setenv("CAMLI_TWITTER_API_KEY", c.twitterAPIKey)
}
setenv("CAMLI_CONFIG_DIR", "config")
setenv("CAMLI_CACHE_DIR", filepath.Join(c.root, "cache"))
setenv("CAMLI_APP_BINDIR", "bin")
return nil
}
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
checkType := func(key string, htype string) {
v := conf.OptionalString(key, "")
if v == "" {
return
}
ct := ld.GetHandlerType(v)
if ct == "" {
err = fmt.Errorf("root handler's %q references non-existant %q", key, v)
} else if ct != htype {
err = fmt.Errorf("root handler's %q references %q of type %q; expected type %q", key, v, ct, htype)
}
}
checkType("searchRoot", "search")
checkType("jsonSignRoot", "jsonsign")
if err != nil {
return
}
username, _ := getUserName()
root := &RootHandler{
BlobRoot: conf.OptionalString("blobRoot", ""),
SearchRoot: conf.OptionalString("searchRoot", ""),
JSONSignRoot: conf.OptionalString("jsonSignRoot", ""),
OwnerName: conf.OptionalString("ownerName", username),
Username: osutil.Username(),
Prefix: ld.MyPrefix(),
}
root.Stealth = conf.OptionalBool("stealth", false)
root.statusRoot = conf.OptionalString("statusRoot", "")
root.helpRoot = conf.OptionalString("helpRoot", "")
if err = conf.Validate(); err != nil {
return
}
if root.BlobRoot != "" {
bs, err := ld.GetStorage(root.BlobRoot)
if err != nil {
return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err)
}
root.Storage = bs
}
if root.JSONSignRoot != "" {
h, _ := ld.GetHandler(root.JSONSignRoot)
if sigh, ok := h.(*signhandler.Handler); ok {
root.sigh = sigh
}
}
root.searchInit = func() {}
if root.SearchRoot != "" {
prefix := root.SearchRoot
if t := ld.GetHandlerType(prefix); t != "search" {
if t == "" {
return nil, fmt.Errorf("root handler's searchRoot of %q is invalid and doesn't refer to a declared handler", prefix)
}
return nil, fmt.Errorf("root handler's searchRoot of %q is of type %q, not %q", prefix, t, "search")
}
root.searchInit = func() {
h, err := ld.GetHandler(prefix)
if err != nil {
log.Fatalf("Error fetching SearchRoot at %q: %v", prefix, err)
}
root.searchHandler = h.(*search.Handler)
root.searchInit = nil
}
}
if pfx, _, _ := ld.FindHandlerByType("importer"); err == nil {
root.importerRoot = pfx
}
return root, nil
}
func (c *serverCmd) setEnvVars() error {
setenv := func(k, v string) {
c.env.Set(k, v)
}
if c.slow {
setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle))
setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency))
}
if c.debug {
setenv("CAMLI_HTTP_DEBUG", "1")
}
user := osutil.Username()
if user == "" {
return errors.New("Could not get username from environment")
}
setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false") // TODO: option to make this true
setenv("CAMLI_DBNAME", "devcamli"+user)
setenv("CAMLI_MYSQL_ENABLED", "false")
setenv("CAMLI_MONGO_ENABLED", "false")
setenv("CAMLI_POSTGRES_ENABLED", "false")
setenv("CAMLI_SQLITE_ENABLED", "false")
setenv("CAMLI_KVINDEX_ENABLED", "false")
switch {
case c.mongo:
setenv("CAMLI_MONGO_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mongo/")
case c.postgres:
setenv("CAMLI_POSTGRES_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-postgres/")
case c.mysql:
setenv("CAMLI_MYSQL_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mysql/")
case c.sqlite:
setenv("CAMLI_SQLITE_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-sqlite/")
if c.camliRoot == "" {
panic("no camliRoot set")
}
setenv("CAMLI_DBNAME", filepath.Join(c.camliRoot, "sqliteindex.db"))
default:
setenv("CAMLI_KVINDEX_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-kv/")
if c.camliRoot == "" {
panic("no camliRoot set")
}
setenv("CAMLI_DBNAME", filepath.Join(c.camliRoot, "kvindex.db"))
}
base := "http://localhost:" + c.port
c.listen = "127.0.0.1:" + c.port
if c.all {
c.listen = "0.0.0.0:" + c.port
if c.hostname == "" {
hostname, err := os.Hostname()
if err != nil {
return fmt.Errorf("Could not get system hostname: %v", err)
}
base = "http://" + hostname + ":" + c.port
} else {
base = "http://" + c.hostname + ":" + c.port
}
}
setenv("CAMLI_TLS", "false")
if c.tls {
base = strings.Replace(base, "http://", "https://", 1)
setenv("CAMLI_TLS", "true")
}
setenv("CAMLI_BASEURL", base)
setenv("CAMLI_DEV_CAMLI_ROOT", camliSrcRoot)
setenv("CAMLI_AUTH", "devauth:pass3179")
fullSuffix := func(name string) string {
return filepath.Join(c.camliRoot, name)
}
suffixes := map[string]string{
"CAMLI_ROOT": fullSuffix("bs"),
"CAMLI_ROOT_SHARD1": fullSuffix("s1"),
"CAMLI_ROOT_SHARD2": fullSuffix("s2"),
"CAMLI_ROOT_REPLICA1": fullSuffix("r1"),
"CAMLI_ROOT_REPLICA2": fullSuffix("r2"),
"CAMLI_ROOT_REPLICA3": fullSuffix("r3"),
"CAMLI_ROOT_CACHE": fullSuffix("cache"),
"CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"),
"CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"),
}
for k, v := range suffixes {
c.makeSuffixdir(v)
setenv(k, v)
}
setenv("CAMLI_PORT", c.port)
setenv("CAMLI_SECRET_RING", filepath.Join(camliSrcRoot,
filepath.FromSlash(defaultSecring)))
setenv("CAMLI_KEYID", defaultKeyID)
if c.flickrAPIKey != "" {
setenv("CAMLI_FLICKR_ENABLED", "true")
setenv("CAMLI_FLICKR_API_KEY", c.flickrAPIKey)
}
setenv("CAMLI_CONFIG_DIR", "config")
return nil
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
obj := jsonconfig.Obj{}
if conf.HTTPS {
if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") {
return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)")
}
if conf.HTTPSCert != "" {
obj["httpsCert"] = conf.HTTPSCert
obj["httpsKey"] = conf.HTTPSKey
} else {
obj["httpsCert"] = osutil.DefaultTLSCert()
obj["httpsKey"] = osutil.DefaultTLSKey()
}
}
if conf.BaseURL != "" {
u, err := url.Parse(conf.BaseURL)
if err != nil {
return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", conf.BaseURL, err)
}
if u.Path != "" && u.Path != "/" {
return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
}
u.Path = ""
obj["baseURL"] = u.String()
}
if conf.Listen != "" {
obj["listen"] = conf.Listen
}
obj["https"] = conf.HTTPS
obj["auth"] = conf.Auth
username := ""
if conf.DBName == "" {
username = osutil.Username()
if username == "" {
return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
}
conf.DBName = "camli" + username
}
var indexerPath string
numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile)
runIndex := conf.RunIndex.Get()
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
case runIndex && numIndexers != 1:
return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
case conf.MySQL != "":
indexerPath = "/index-mysql/"
case conf.PostgreSQL != "":
indexerPath = "/index-postgres/"
case conf.Mongo != "":
indexerPath = "/index-mongo/"
case conf.SQLite != "":
indexerPath = "/index-sqlite/"
case conf.KVFile != "":
indexerPath = "/index-kv/"
}
entity, err := jsonsign.EntityFromSecring(conf.Identity, conf.IdentitySecretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
nolocaldisk := conf.BlobPath == ""
if nolocaldisk {
if conf.S3 == "" && conf.GoogleCloudStorage == "" {
return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
}
if conf.S3 != "" && conf.GoogleCloudStorage != "" {
return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
}
}
if conf.ShareHandler && conf.ShareHandlerPath == "" {
conf.ShareHandlerPath = "/share/"
}
prefixesParams := &configPrefixesParams{
secretRing: conf.IdentitySecretRing,
keyId: conf.Identity,
indexerPath: indexerPath,
blobPath: conf.BlobPath,
packBlobs: conf.PackBlobs,
searchOwner: blob.SHA1FromString(armoredPublicKey),
shareHandlerPath: conf.ShareHandlerPath,
flickr: conf.Flickr,
memoryIndex: conf.MemoryIndex.Get(),
}
prefixes := genLowLevelPrefixes(prefixesParams, conf.OwnerName)
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
cacheDir = filepath.Join(tempDir(), "camli-cache")
} else {
cacheDir = filepath.Join(conf.BlobPath, "cache")
}
if !noMkdir {
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
}
}
published := []interface{}{}
if len(conf.Publish) > 0 {
if !runIndex {
return nil, fmt.Errorf("publishing requires an index")
}
published, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
if runIndex {
addUIConfig(prefixesParams, prefixes, "/ui/", published, conf.SourceRoot)
}
if conf.MySQL != "" {
addMySQLConfig(prefixes, conf.DBName, conf.MySQL)
}
if conf.PostgreSQL != "" {
addPostgresConfig(prefixes, conf.DBName, conf.PostgreSQL)
}
if conf.Mongo != "" {
addMongoConfig(prefixes, conf.DBName, conf.Mongo)
}
if conf.SQLite != "" {
addSQLiteConfig(prefixes, conf.SQLite)
}
if conf.KVFile != "" {
addKVConfig(prefixes, conf.KVFile)
}
if conf.S3 != "" {
if err := addS3Config(prefixesParams, prefixes, conf.S3); err != nil {
return nil, err
}
}
if conf.GoogleDrive != "" {
if err := addGoogleDriveConfig(prefixes, conf.GoogleDrive); err != nil {
return nil, err
}
}
if conf.GoogleCloudStorage != "" {
if err := addGoogleCloudStorageConfig(prefixes, conf.GoogleCloudStorage); err != nil {
return nil, err
}
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
}
return lowLevelConf, nil
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
var (
baseURL = conf.OptionalString("baseURL", "")
listen = conf.OptionalString("listen", "")
auth = conf.RequiredString("auth")
keyId = conf.RequiredString("identity")
secretRing = conf.RequiredString("identitySecretRing")
tlsOn = conf.OptionalBool("https", false)
tlsCert = conf.OptionalString("HTTPSCertFile", "")
tlsKey = conf.OptionalString("HTTPSKeyFile", "")
// Blob storage options
blobPath = conf.OptionalString("blobPath", "")
packBlobs = conf.OptionalBool("packBlobs", false) // use diskpacked instead of the default filestorage
s3 = conf.OptionalString("s3", "") // "access_key_id:secret_access_key:bucket[:hostname]"
googlecloudstorage = conf.OptionalString("googlecloudstorage", "") // "clientId:clientSecret:refreshToken:bucket"
googledrive = conf.OptionalString("googledrive", "") // "clientId:clientSecret:refreshToken:parentId"
swift = conf.OptionalString("swift", "") // "tenant:secret:container:auth_url"
// Enable the share handler. If true, and shareHandlerPath is empty,
// then shareHandlerPath defaults to "/share/".
shareHandler = conf.OptionalBool("shareHandler", false)
// URL prefix for the share handler. If set, overrides shareHandler.
shareHandlerPath = conf.OptionalString("shareHandlerPath", "")
// Index options
memoryIndex = conf.OptionalBool("memoryIndex", true) // copy disk-based index to memory on start-up
runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc.
dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo
mysql = conf.OptionalString("mysql", "")
postgres = conf.OptionalString("postgres", "")
mongo = conf.OptionalString("mongo", "")
sqliteFile = conf.OptionalString("sqlite", "")
kvFile = conf.OptionalString("kvIndexFile", "")
// Importer options
flickr = conf.OptionalString("flickr", "")
_ = conf.OptionalList("replicateTo")
publish = conf.OptionalObject("publish")
// alternative source tree, to override the embedded ui and/or closure resources.
// If non empty, the ui files will be expected at
// sourceRoot + "/server/camlistored/ui" and the closure library at
// sourceRoot + "/third_party/closure/lib"
// Also used by the publish handler.
sourceRoot = conf.OptionalString("sourceRoot", "")
ownerName = conf.OptionalString("ownerName", "")
)
if err := conf.Validate(); err != nil {
return nil, err
}
obj := jsonconfig.Obj{}
if tlsOn {
if (tlsCert != "") != (tlsKey != "") {
return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
}
if tlsCert != "" {
obj["TLSCertFile"] = tlsCert
obj["TLSKeyFile"] = tlsKey
} else {
obj["TLSCertFile"] = osutil.DefaultTLSCert()
obj["TLSKeyFile"] = osutil.DefaultTLSKey()
}
}
if baseURL != "" {
u, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", baseURL, err)
}
if u.Path != "" && u.Path != "/" {
return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
}
u.Path = ""
obj["baseURL"] = u.String()
}
if listen != "" {
obj["listen"] = listen
}
obj["https"] = tlsOn
obj["auth"] = auth
username := ""
if dbname == "" {
username = osutil.Username()
if username == "" {
return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
}
dbname = "camli" + username
}
var indexerPath string
numIndexers := numSet(mongo, mysql, postgres, sqliteFile, kvFile)
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
case runIndex && numIndexers != 1:
return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
case mysql != "":
indexerPath = "/index-mysql/"
case postgres != "":
indexerPath = "/index-postgres/"
case mongo != "":
indexerPath = "/index-mongo/"
case sqliteFile != "":
indexerPath = "/index-sqlite/"
case kvFile != "":
indexerPath = "/index-kv/"
}
entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
nolocaldisk := blobPath == ""
if nolocaldisk {
if s3 == "" && googlecloudstorage == "" {
return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
}
if s3 != "" && googlecloudstorage != "" {
return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
}
}
if shareHandler && shareHandlerPath == "" {
shareHandlerPath = "/share/"
}
prefixesParams := &configPrefixesParams{
secretRing: secretRing,
keyId: keyId,
indexerPath: indexerPath,
blobPath: blobPath,
packBlobs: packBlobs,
searchOwner: blob.SHA1FromString(armoredPublicKey),
shareHandlerPath: shareHandlerPath,
flickr: flickr,
memoryIndex: memoryIndex,
}
prefixes := genLowLevelPrefixes(prefixesParams, ownerName)
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
cacheDir = filepath.Join(tempDir(), "camli-cache")
} else {
cacheDir = filepath.Join(blobPath, "cache")
}
if !noMkdir {
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
}
}
published := []interface{}{}
if len(publish) > 0 {
if !runIndex {
return nil, fmt.Errorf("publishing requires an index")
}
published, err = addPublishedConfig(prefixes, publish, sourceRoot)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
if runIndex {
addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot)
}
if mysql != "" {
addMySQLConfig(prefixes, dbname, mysql)
}
if postgres != "" {
addPostgresConfig(prefixes, dbname, postgres)
}
if mongo != "" {
addMongoConfig(prefixes, dbname, mongo)
}
if sqliteFile != "" {
addSQLiteConfig(prefixes, sqliteFile)
}
if kvFile != "" {
addKVConfig(prefixes, kvFile)
}
if s3 != "" {
if err := addS3Config(prefixesParams, prefixes, s3); err != nil {
return nil, err
}
}
if googledrive != "" {
if err := addGoogleDriveConfig(prefixes, googledrive); err != nil {
return nil, err
}
}
if googlecloudstorage != "" {
if err := addGoogleCloudStorageConfig(prefixes, googlecloudstorage); err != nil {
return nil, err
}
}
if swift != "" {
if err := addSwiftConfig(prefixesParams, prefixes, swift); err != nil {
return nil, err
}
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
configPath: conf.configPath,
}
return lowLevelConf, nil
}
func (c *serverCmd) setEnvVars() error {
if c.slow {
setenv("DEV_THROTTLE_KBPS", fmt.Sprintf("%d", c.throttle))
setenv("DEV_THROTTLE_LATENCY_MS", fmt.Sprintf("%d", c.latency))
}
if c.debug {
setenv("CAMLI_HTTP_DEBUG", "1")
}
user := osutil.Username()
if user == "" {
return errors.New("Could not get username from environment")
}
setenv("CAMLI_DBNAME", "devcamli"+user)
setenv("CAMLI_MYSQL_ENABLED", "false")
setenv("CAMLI_MONGO_ENABLED", "false")
setenv("CAMLI_POSTGRES_ENABLED", "false")
setenv("CAMLI_KVINDEX_ENABLED", "false")
setenv("CAMLI_MEMINDEX_ENABLED", "false")
switch {
case c.mongo:
setenv("CAMLI_MONGO_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mongo/")
case c.postgres:
setenv("CAMLI_POSTGRES_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-postgres/")
case c.mysql:
setenv("CAMLI_MYSQL_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mysql/")
case c.memindex:
setenv("CAMLI_MEMINDEX_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-mem/")
default:
setenv("CAMLI_KVINDEX_ENABLED", "true")
setenv("CAMLI_INDEXER_PATH", "/index-kv/")
if c.camliRoot == "" {
panic("no camliRoot set")
}
setenv("CAMLI_KVINDEX_FILE", filepath.Join(c.camliRoot, "kvindex.db"))
}
base := "http://localhost:" + c.port
c.listen = "127.0.0.1:" + c.port
if c.all {
c.listen = "0.0.0.0:" + c.port
if c.hostname == "" {
hostname, err := os.Hostname()
if err != nil {
return fmt.Errorf("Could not get system hostname: %v", err)
}
base = "http://" + hostname + ":" + c.port
} else {
base = "http://" + c.hostname + ":" + c.port
}
}
setenv("CAMLI_TLS", "false")
if c.tls {
base = strings.Replace(base, "http://", "https://", 1)
setenv("CAMLI_TLS", "true")
}
setenv("CAMLI_BASEURL", base)
setenv("CAMLI_DEV_CAMLI_ROOT", c.camliSrcRoot)
setenv("CAMLI_AUTH", "devauth:pass"+c.port)
fullSuffix := func(name string) string {
return filepath.Join(c.camliRoot, name)
}
suffixes := map[string]string{
"CAMLI_ROOT": fullSuffix("bs"),
"CAMLI_ROOT_SHARD1": fullSuffix("s1"),
"CAMLI_ROOT_SHARD2": fullSuffix("s2"),
"CAMLI_ROOT_REPLICA1": fullSuffix("r1"),
"CAMLI_ROOT_REPLICA2": fullSuffix("r2"),
"CAMLI_ROOT_REPLICA3": fullSuffix("r3"),
"CAMLI_ROOT_CACHE": fullSuffix("cache"),
"CAMLI_ROOT_ENCMETA": fullSuffix("encmeta"),
"CAMLI_ROOT_ENCBLOB": fullSuffix("encblob"),
}
for k, v := range suffixes {
c.makeSuffixdir(v)
setenv(k, v)
}
setenv("CAMLI_PORT", c.port)
setenv("CAMLI_SECRET_RING", filepath.Join(c.camliSrcRoot,
filepath.FromSlash("pkg/jsonsign/testdata/test-secring.gpg")))
return nil
}