func (c *Client) initSignerPublicKeyBlobref() {
if c.paramsOnly {
log.Print("client: paramsOnly set; cannot get public key from config or env vars.")
return
}
keyId := os.Getenv("CAMLI_KEYID")
if keyId == "" {
configOnce.Do(parseConfig)
keyId = config.Identity
if keyId == "" {
log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"camput init\"?", osutil.UserClientConfigPath())
}
}
keyRing := c.SecretRingFile()
if !fileExists(keyRing) {
log.Fatalf("Could not find keyId %q, because secret ring file %q does not exist.", keyId, keyRing)
}
entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
if err != nil {
log.Fatalf("Couldn't find keyId %q in secret ring %v: %v", keyId, keyRing, err)
}
armored, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
log.Fatalf("Error serializing public key: %v", err)
}
c.signerPublicKeyRef = blob.SHA1FromString(armored)
c.publicKeyArmored = armored
}
func (e *Env) SetCamdevVars(altkey bool) {
e.Set("CAMLI_CONFIG_DIR", filepath.Join("config", "dev-client-dir"))
e.Set("CAMLI_AUTH", "userpass:camlistore:pass3179")
e.Set("CAMLI_DEV_KEYBLOBS", filepath.FromSlash("config/dev-client-dir/keyblobs"))
secring := defaultSecring
identity := defaultIdentity
if altkey {
secring = filepath.FromSlash("pkg/jsonsign/testdata/password-foo-secring.gpg")
identity = "C7C3E176"
println("**\n** Note: password is \"foo\"\n**\n")
} else {
if *flagSecretRing != "" {
secring = *flagSecretRing
}
if *flagIdentity != "" {
identity = *flagIdentity
}
}
entity, err := jsonsign.EntityFromSecring(identity, secring)
if err != nil {
panic(err)
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
panic(err)
}
pubKeyRef := blob.SHA1FromString(armoredPublicKey)
e.Set("CAMLI_SECRET_RING", secring)
e.Set("CAMLI_KEYID", identity)
e.Set("CAMLI_PUBKEY_BLOBREF", pubKeyRef.String())
}
func getSignerPublicKeyBlobref() (signerRef blob.Ref, armored string, ok bool) {
configOnce.Do(parseConfig)
key := "keyId"
keyId, ok := config[key].(string)
if !ok {
log.Printf("No key %q in JSON configuration file %q; have you run \"camput init\"?", key, osutil.UserClientConfigPath())
return
}
keyRing, hasKeyRing := config["secretRing"].(string)
if !hasKeyRing {
if fn := osutil.IdentitySecretRing(); fileExists(fn) {
keyRing = fn
} else if fn := jsonsign.DefaultSecRingPath(); fileExists(fn) {
keyRing = fn
} else {
log.Printf("Couldn't find keyId %q; no 'secretRing' specified in config file, and no standard secret ring files exist.")
return
}
}
entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
if err != nil {
log.Printf("Couldn't find keyId %q in secret ring: %v", keyId, err)
return
}
armored, err = jsonsign.ArmoredPublicKey(entity)
if err != nil {
log.Printf("Error serializing public key: %v", err)
return
}
// TODO(mpl): integrate with getSelfPubKeyDir if possible.
selfPubKeyDir, ok := config["selfPubKeyDir"].(string)
if !ok {
selfPubKeyDir = osutil.KeyBlobsDir()
log.Printf("No 'selfPubKeyDir' defined in %q, defaulting to %v", osutil.UserClientConfigPath(), selfPubKeyDir)
}
fi, err := os.Stat(selfPubKeyDir)
if err != nil || !fi.IsDir() {
log.Printf("selfPubKeyDir of %q doesn't exist or not a directory", selfPubKeyDir)
return
}
br := blob.SHA1FromString(armored)
pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
fi, err = os.Stat(pubFile)
if err != nil {
err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
if err != nil {
log.Printf("Error writing public key to %q: %v", pubFile, err)
return
}
}
return br, armored, true
}
// TODO: move to config package?
func SignerPublicKeyBlobref() *blobref.BlobRef {
configOnce.Do(parseConfig)
key := "keyId"
keyId, ok := config[key].(string)
if !ok {
log.Printf("No key %q in JSON configuration file %q; have you run \"camput init\"?", key, ConfigFilePath())
return nil
}
keyRing, hasKeyRing := config["secretRing"].(string)
if !hasKeyRing {
if fn := osutil.IdentitySecretRing(); fileExists(fn) {
keyRing = fn
} else if fn := jsonsign.DefaultSecRingPath(); fileExists(fn) {
keyRing = fn
} else {
log.Printf("Couldn't find keyId %q; no 'secretRing' specified in config file, and no standard secret ring files exist.")
return nil
}
}
entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
if err != nil {
log.Printf("Couldn't find keyId %q in secret ring: %v", keyId, err)
return nil
}
armored, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
log.Printf("Error serializing public key: %v", err)
return nil
}
selfPubKeyDir, ok := config["selfPubKeyDir"].(string)
if !ok {
log.Printf("No 'selfPubKeyDir' defined in %q", ConfigFilePath())
return nil
}
fi, err := os.Stat(selfPubKeyDir)
if err != nil || !fi.IsDir() {
log.Printf("selfPubKeyDir of %q doesn't exist or not a directory", selfPubKeyDir)
return nil
}
br := blobref.SHA1FromString(armored)
pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
log.Printf("key file: %q", pubFile)
fi, err = os.Stat(pubFile)
if err != nil {
err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
if err != nil {
log.Printf("Error writing public key to %q: %v", pubFile, err)
return nil
}
}
return br
}
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
pubKeyDestPrefix := conf.OptionalString("publicKeyDest", "")
// either a short form ("26F5ABDA") or one the longer forms.
keyId := conf.RequiredString("keyId")
h := &Handler{
secretRing: conf.OptionalString("secretRing", ""),
}
var err error
if err = conf.Validate(); err != nil {
return nil, err
}
h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(h.entity)
ms := new(blobref.MemoryStore)
h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, armoredPublicKey)
if err != nil {
return nil, err
}
h.pubKeyFetcher = ms
if pubKeyDestPrefix != "" {
sto, err := ld.GetStorage(pubKeyDestPrefix)
if err != nil {
return nil, err
}
h.pubKeyDest = sto
if sto != nil {
if ctxReq, ok := ld.GetRequestContext(); ok {
if w, ok := sto.(blobserver.ContextWrapper); ok {
sto = w.WrapContext(ctxReq)
}
}
err := h.uploadPublicKey(sto, armoredPublicKey)
if err != nil {
return nil, fmt.Errorf("Error seeding self public key in storage: %v", err)
}
}
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &gethandler.Handler{
Fetcher: ms,
AllowGlobalAccess: true, // just public keys
}
return h, nil
}
func (c *initCmd) getPublicKeyArmored() ([]byte, error) {
entity, err := jsonsign.EntityFromSecring(c.keyId, c.secretRing)
if err != nil {
return nil, fmt.Errorf("Could not find keyId %v in ring %v: %v", c.keyId, c.secretRing, err)
}
pubArmor, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, fmt.Errorf("failed to export armored public key ID %q from %v: %v", c.keyId, c.secretRing, err)
}
return []byte(pubArmor), nil
}
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
pubKeyDestPrefix := conf.OptionalString("publicKeyDest", "")
// either a short form ("26F5ABDA") or one the longer forms.
keyId := conf.RequiredString("keyId")
h := &Handler{
secretRing: conf.OptionalString("secretRing", ""),
}
var err error
if err = conf.Validate(); err != nil {
return nil, err
}
h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(h.entity)
ms := new(blob.MemoryStore)
h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, armoredPublicKey)
if err != nil {
return nil, err
}
h.pubKeyFetcher = ms
if pubKeyDestPrefix != "" {
sto, err := ld.GetStorage(pubKeyDestPrefix)
if err != nil {
return nil, err
}
h.pubKeyDest = sto
if sto != nil {
err := h.uploadPublicKey(sto, armoredPublicKey)
if err != nil {
return nil, fmt.Errorf("Error seeding self public key in storage: %v", err)
}
}
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &gethandler.Handler{
Fetcher: ms,
}
h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(armoredPublicKey), h.entity)
if err != nil {
return nil, err
}
return h, nil
}
// Error is errNoOwner if no identity configured
func (b *lowBuilder) searchOwner() (br blob.Ref, err error) {
if b.high.Identity == "" {
return br, errNoOwner
}
entity, err := jsonsign.EntityFromSecring(b.high.Identity, b.high.IdentitySecretRing)
if err != nil {
return br, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return br, err
}
return blob.SHA1FromString(armoredPublicKey), nil
}
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
// either a short form ("26F5ABDA") or one the longer forms.
keyId = conf.RequiredString("keyId")
pubKeyDestPrefix = conf.OptionalString("publicKeyDest", "")
secretRing = conf.OptionalString("secretRing", "")
)
if err := conf.Validate(); err != nil {
return nil, err
}
h := &Handler{
secretRing: secretRing,
}
var err error
h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
if err != nil {
return nil, err
}
h.pubKey, err = jsonsign.ArmoredPublicKey(h.entity)
ms := &memory.Storage{}
h.pubKeyBlobRef = blob.SHA1FromString(h.pubKey)
if _, err := ms.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(h.pubKey)); err != nil {
return nil, fmt.Errorf("could not store pub key blob: %v", err)
}
h.pubKeyFetcher = ms
if pubKeyDestPrefix != "" {
sto, err := ld.GetStorage(pubKeyDestPrefix)
if err != nil {
return nil, err
}
h.pubKeyDest = sto
}
h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
h.pubKeyHandler = &gethandler.Handler{
Fetcher: ms,
}
h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(h.pubKey), h.entity)
if err != nil {
return nil, err
}
return h, nil
}
// NewSigner returns an Signer given an armored public key's blobref,
// its armored content, and its associated private key entity.
// The privateKeySource must be either an *openpgp.Entity or a string filename to a secret key.
func NewSigner(pubKeyRef blob.Ref, armoredPubKey io.Reader, privateKeySource interface{}) (*Signer, error) {
hash := pubKeyRef.Hash()
keyId, armoredPubKeyString, err := jsonsign.ParseArmoredPublicKey(io.TeeReader(armoredPubKey, hash))
if err != nil {
return nil, err
}
if !pubKeyRef.HashMatches(hash) {
return nil, fmt.Errorf("pubkey ref of %v doesn't match provided armored public key", pubKeyRef)
}
var privateKey *openpgp.Entity
switch v := privateKeySource.(type) {
case *openpgp.Entity:
privateKey = v
case string:
privateKey, err = jsonsign.EntityFromSecring(keyId, v)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("invalid privateKeySource type %T", v)
}
if privateKey == nil {
return nil, errors.New("nil privateKey")
}
return &Signer{
keyId: keyId,
pubref: pubKeyRef,
privEntity: privateKey,
baseSigReq: jsonsign.SignRequest{
ServerMode: true, // shouldn't matter, since we're supplying the rest of the fields
Fetcher: memoryBlobFetcher{
pubKeyRef: func() (uint32, io.ReadCloser) {
return uint32(len(armoredPubKeyString)), ioutil.NopCloser(strings.NewReader(armoredPubKeyString))
},
},
EntityFetcher: entityFetcherFunc(func(wantKeyId string) (*openpgp.Entity, error) {
if privateKey.PrivateKey.KeyIdString() != wantKeyId &&
privateKey.PrivateKey.KeyIdShortString() != wantKeyId {
return nil, fmt.Errorf("jsonsign code unexpectedly requested keyId %q; only have %q",
wantKeyId, keyId)
}
return privateKey, nil
}),
},
}, nil
}
func setCamdevVarsFor(e *Env, altkey bool) {
var setenv func(string, string) error
if e != nil {
setenv = func(k, v string) error { e.Set(k, v); return nil }
} else {
setenv = os.Setenv
}
setenv("CAMLI_AUTH", "userpass:camlistore:pass3179")
// env values for clients. server will overwrite them anyway in its setEnvVars.
root, err := rootInTmpDir()
if err != nil {
log.Fatal(err)
}
setenv("CAMLI_CACHE_DIR", filepath.Join(root, "client", "cache"))
setenv("CAMLI_CONFIG_DIR", filepath.Join("config", "dev-client-dir"))
secring := defaultSecring
identity := defaultIdentity
if altkey {
secring = filepath.FromSlash("pkg/jsonsign/testdata/password-foo-secring.gpg")
identity = "C7C3E176"
println("**\n** Note: password is \"foo\"\n**\n")
} else {
if *flagSecretRing != "" {
secring = *flagSecretRing
}
if *flagIdentity != "" {
identity = *flagIdentity
}
}
entity, err := jsonsign.EntityFromSecring(identity, secring)
if err != nil {
panic(err)
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
panic(err)
}
pubKeyRef := blob.SHA1FromString(armoredPublicKey)
setenv("CAMLI_SECRET_RING", secring)
setenv("CAMLI_KEYID", identity)
setenv("CAMLI_PUBKEY_BLOBREF", pubKeyRef.String())
setenv("CAMLI_KV_VERIFY", "true")
}
func (c *initCmd) getPublicKeyArmoredFromFile(secretRingFileName, keyId string) (b []byte, err error) {
entity, err := jsonsign.EntityFromSecring(keyId, secretRingFileName)
if err == nil {
pubArmor, err := jsonsign.ArmoredPublicKey(entity)
if err == nil {
return []byte(pubArmor), nil
}
}
b, err = exec.Command("gpg", "--export", "--armor", keyId).Output()
if err != nil {
return nil, fmt.Errorf("Error running gpg to export public key %q: %v", keyId, err)
}
if len(b) == 0 {
return nil, fmt.Errorf("gpg export of public key %q was empty.", keyId)
}
return b, nil
}
func (c *Client) initSignerPublicKeyBlobref() {
configOnce.Do(parseConfig)
keyId := config.identity
if keyId == "" {
log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"camput init\"?", osutil.UserClientConfigPath())
}
keyRing := c.SecretRingFile()
if !fileExists(keyRing) {
log.Fatalf("Could not find keyId %q, because secret ring file %q does not exist.", keyId, keyRing)
}
entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
if err != nil {
log.Fatalf("Couldn't find keyId %q in secret ring %v: %v", keyId, keyRing, err)
}
armored, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
log.Fatalf("Error serializing public key: %v", err)
}
// TODO(mpl): completely get rid of it if possible
// http://camlistore.org/issue/377
selfPubKeyDir := osutil.KeyBlobsDir()
fi, err := os.Stat(selfPubKeyDir)
if err != nil || !fi.IsDir() {
log.Fatalf("selfPubKeyDir as %q doesn't exist or not a directory", selfPubKeyDir)
}
br := blob.SHA1FromString(armored)
pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
fi, err = os.Stat(pubFile)
if err != nil {
if !os.IsNotExist(err) {
log.Fatalf("Could not stat %q: %v", pubFile, err)
}
err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
if err != nil {
log.Fatalf("Error writing public key to %q: %v", pubFile, err)
}
}
c.signerPublicKeyRef = br
c.publicKeyArmored = armored
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
var (
baseURL = conf.OptionalString("baseURL", "")
listen = conf.OptionalString("listen", "")
auth = conf.RequiredString("auth")
keyId = conf.RequiredString("identity")
secretRing = conf.RequiredString("identitySecretRing")
tlsOn = conf.OptionalBool("https", false)
tlsCert = conf.OptionalString("HTTPSCertFile", "")
tlsKey = conf.OptionalString("HTTPSKeyFile", "")
// Blob storage options
blobPath = conf.OptionalString("blobPath", "")
s3 = conf.OptionalString("s3", "") // "access_key_id:secret_access_key:bucket"
shareHandler = conf.OptionalBool("shareHandler", true) // enable the share handler
// Index options
runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc.
dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo
mysql = conf.OptionalString("mysql", "")
postgres = conf.OptionalString("postgres", "")
memIndex = conf.OptionalBool("memIndex", false)
mongo = conf.OptionalString("mongo", "")
sqliteFile = conf.OptionalString("sqlite", "")
_ = conf.OptionalList("replicateTo")
publish = conf.OptionalObject("publish")
)
if err := conf.Validate(); err != nil {
return nil, err
}
obj := jsonconfig.Obj{}
if tlsOn {
if (tlsCert != "") != (tlsKey != "") {
return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
}
if tlsCert != "" {
obj["TLSCertFile"] = tlsCert
obj["TLSKeyFile"] = tlsKey
} else {
obj["TLSCertFile"] = DefaultTLSCert
obj["TLSKeyFile"] = DefaultTLSKey
}
}
if baseURL != "" {
if strings.HasSuffix(baseURL, "/") {
baseURL = baseURL[:len(baseURL)-1]
}
obj["baseURL"] = baseURL
}
if listen != "" {
obj["listen"] = listen
}
obj["https"] = tlsOn
obj["auth"] = auth
if dbname == "" {
username := os.Getenv("USER")
if username == "" {
return nil, fmt.Errorf("USER env var not set; needed to define dbname")
}
dbname = "camli" + username
}
var indexerPath string
numIndexers := numSet(mongo, mysql, postgres, sqliteFile, memIndex)
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless wantIndex is set to false, you must specify an index option (mongo, mysql, postgres, sqlite, memIndex).")
case runIndex && numIndexers != 1:
return nil, fmt.Errorf("With wantIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite, memIndex).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With wantIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite, memIndex.")
case mysql != "":
indexerPath = "/index-mysql/"
case postgres != "":
indexerPath = "/index-postgres/"
case mongo != "":
indexerPath = "/index-mongo/"
case sqliteFile != "":
indexerPath = "/index-sqlite/"
case memIndex:
indexerPath = "/index-mem/"
}
entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
nolocaldisk := blobPath == ""
if nolocaldisk && s3 == "" {
return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 configured for a blobserver.")
}
prefixesParams := &configPrefixesParams{
secretRing: secretRing,
keyId: keyId,
indexerPath: indexerPath,
blobPath: blobPath,
searchOwner: blobref.SHA1FromString(armoredPublicKey),
shareHandler: shareHandler,
}
prefixes := genLowLevelPrefixes(prefixesParams)
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
cacheDir = filepath.Join(tempDir(), "camli-cache")
} else {
cacheDir = filepath.Join(blobPath, "/cache")
}
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
}
published := []interface{}{}
if len(publish) > 0 {
if !runIndex {
return nil, fmt.Errorf("publishing requires an index")
}
published, err = addPublishedConfig(prefixes, publish)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
if runIndex {
addUIConfig(prefixes, "/ui/", published)
}
if mysql != "" {
addMySQLConfig(prefixes, dbname, mysql)
}
if postgres != "" {
addPostgresConfig(prefixes, dbname, postgres)
}
if mongo != "" {
addMongoConfig(prefixes, dbname, mongo)
}
if sqliteFile != "" {
addSQLiteConfig(prefixes, sqliteFile)
}
if s3 != "" {
if err := addS3Config(prefixes, s3); err != nil {
return nil, err
}
}
if indexerPath == "/index-mem/" {
addMemindexConfig(prefixes)
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
configPath: conf.configPath,
}
return lowLevelConf, nil
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
obj := jsonconfig.Obj{}
if conf.HTTPS {
if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") {
return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)")
}
if conf.HTTPSCert != "" {
obj["httpsCert"] = conf.HTTPSCert
obj["httpsKey"] = conf.HTTPSKey
} else {
obj["httpsCert"] = osutil.DefaultTLSCert()
obj["httpsKey"] = osutil.DefaultTLSKey()
}
}
if conf.BaseURL != "" {
u, err := url.Parse(conf.BaseURL)
if err != nil {
return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", conf.BaseURL, err)
}
if u.Path != "" && u.Path != "/" {
return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
}
u.Path = ""
obj["baseURL"] = u.String()
}
if conf.Listen != "" {
obj["listen"] = conf.Listen
}
obj["https"] = conf.HTTPS
obj["auth"] = conf.Auth
username := ""
if conf.DBName == "" {
username = osutil.Username()
if username == "" {
return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
}
conf.DBName = "camli" + username
}
var indexerPath string
numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile)
runIndex := conf.RunIndex.Get()
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
case runIndex && numIndexers != 1:
return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
case conf.MySQL != "":
indexerPath = "/index-mysql/"
case conf.PostgreSQL != "":
indexerPath = "/index-postgres/"
case conf.Mongo != "":
indexerPath = "/index-mongo/"
case conf.SQLite != "":
indexerPath = "/index-sqlite/"
case conf.KVFile != "":
indexerPath = "/index-kv/"
}
entity, err := jsonsign.EntityFromSecring(conf.Identity, conf.IdentitySecretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
nolocaldisk := conf.BlobPath == ""
if nolocaldisk {
if conf.S3 == "" && conf.GoogleCloudStorage == "" {
return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
}
if conf.S3 != "" && conf.GoogleCloudStorage != "" {
return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
}
}
if conf.ShareHandler && conf.ShareHandlerPath == "" {
conf.ShareHandlerPath = "/share/"
}
prefixesParams := &configPrefixesParams{
secretRing: conf.IdentitySecretRing,
keyId: conf.Identity,
indexerPath: indexerPath,
blobPath: conf.BlobPath,
packBlobs: conf.PackBlobs,
searchOwner: blob.SHA1FromString(armoredPublicKey),
shareHandlerPath: conf.ShareHandlerPath,
flickr: conf.Flickr,
memoryIndex: conf.MemoryIndex.Get(),
}
prefixes := genLowLevelPrefixes(prefixesParams, conf.OwnerName)
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
cacheDir = filepath.Join(tempDir(), "camli-cache")
} else {
cacheDir = filepath.Join(conf.BlobPath, "cache")
}
if !noMkdir {
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
}
}
published := []interface{}{}
if len(conf.Publish) > 0 {
if !runIndex {
return nil, fmt.Errorf("publishing requires an index")
}
published, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
if runIndex {
addUIConfig(prefixesParams, prefixes, "/ui/", published, conf.SourceRoot)
}
if conf.MySQL != "" {
addMySQLConfig(prefixes, conf.DBName, conf.MySQL)
}
if conf.PostgreSQL != "" {
addPostgresConfig(prefixes, conf.DBName, conf.PostgreSQL)
}
if conf.Mongo != "" {
addMongoConfig(prefixes, conf.DBName, conf.Mongo)
}
if conf.SQLite != "" {
addSQLiteConfig(prefixes, conf.SQLite)
}
if conf.KVFile != "" {
addKVConfig(prefixes, conf.KVFile)
}
if conf.S3 != "" {
if err := addS3Config(prefixesParams, prefixes, conf.S3); err != nil {
return nil, err
}
}
if conf.GoogleDrive != "" {
if err := addGoogleDriveConfig(prefixes, conf.GoogleDrive); err != nil {
return nil, err
}
}
if conf.GoogleCloudStorage != "" {
if err := addGoogleCloudStorageConfig(prefixes, conf.GoogleCloudStorage); err != nil {
return nil, err
}
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
}
return lowLevelConf, nil
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
var (
baseURL = conf.OptionalString("baseURL", "")
listen = conf.OptionalString("listen", "")
auth = conf.RequiredString("auth")
keyId = conf.RequiredString("identity")
secretRing = conf.RequiredString("identitySecretRing")
tlsOn = conf.OptionalBool("https", false)
tlsCert = conf.OptionalString("HTTPSCertFile", "")
tlsKey = conf.OptionalString("HTTPSKeyFile", "")
// Blob storage options
blobPath = conf.OptionalString("blobPath", "")
packBlobs = conf.OptionalBool("packBlobs", false) // use diskpacked instead of the default filestorage
s3 = conf.OptionalString("s3", "") // "access_key_id:secret_access_key:bucket[:hostname]"
googlecloudstorage = conf.OptionalString("googlecloudstorage", "") // "clientId:clientSecret:refreshToken:bucket"
googledrive = conf.OptionalString("googledrive", "") // "clientId:clientSecret:refreshToken:parentId"
swift = conf.OptionalString("swift", "") // "tenant:secret:container:auth_url"
// Enable the share handler. If true, and shareHandlerPath is empty,
// then shareHandlerPath defaults to "/share/".
shareHandler = conf.OptionalBool("shareHandler", false)
// URL prefix for the share handler. If set, overrides shareHandler.
shareHandlerPath = conf.OptionalString("shareHandlerPath", "")
// Index options
memoryIndex = conf.OptionalBool("memoryIndex", true) // copy disk-based index to memory on start-up
runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc.
dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo
mysql = conf.OptionalString("mysql", "")
postgres = conf.OptionalString("postgres", "")
mongo = conf.OptionalString("mongo", "")
sqliteFile = conf.OptionalString("sqlite", "")
kvFile = conf.OptionalString("kvIndexFile", "")
// Importer options
flickr = conf.OptionalString("flickr", "")
_ = conf.OptionalList("replicateTo")
publish = conf.OptionalObject("publish")
// alternative source tree, to override the embedded ui and/or closure resources.
// If non empty, the ui files will be expected at
// sourceRoot + "/server/camlistored/ui" and the closure library at
// sourceRoot + "/third_party/closure/lib"
// Also used by the publish handler.
sourceRoot = conf.OptionalString("sourceRoot", "")
ownerName = conf.OptionalString("ownerName", "")
)
if err := conf.Validate(); err != nil {
return nil, err
}
obj := jsonconfig.Obj{}
if tlsOn {
if (tlsCert != "") != (tlsKey != "") {
return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
}
if tlsCert != "" {
obj["TLSCertFile"] = tlsCert
obj["TLSKeyFile"] = tlsKey
} else {
obj["TLSCertFile"] = osutil.DefaultTLSCert()
obj["TLSKeyFile"] = osutil.DefaultTLSKey()
}
}
if baseURL != "" {
u, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", baseURL, err)
}
if u.Path != "" && u.Path != "/" {
return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
}
u.Path = ""
obj["baseURL"] = u.String()
}
if listen != "" {
obj["listen"] = listen
}
obj["https"] = tlsOn
obj["auth"] = auth
username := ""
if dbname == "" {
username = osutil.Username()
if username == "" {
return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
}
dbname = "camli" + username
}
var indexerPath string
numIndexers := numSet(mongo, mysql, postgres, sqliteFile, kvFile)
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
case runIndex && numIndexers != 1:
return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
case mysql != "":
indexerPath = "/index-mysql/"
case postgres != "":
indexerPath = "/index-postgres/"
case mongo != "":
indexerPath = "/index-mongo/"
case sqliteFile != "":
indexerPath = "/index-sqlite/"
case kvFile != "":
indexerPath = "/index-kv/"
}
entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
nolocaldisk := blobPath == ""
if nolocaldisk {
if s3 == "" && googlecloudstorage == "" {
return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
}
if s3 != "" && googlecloudstorage != "" {
return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
}
}
if shareHandler && shareHandlerPath == "" {
shareHandlerPath = "/share/"
}
prefixesParams := &configPrefixesParams{
secretRing: secretRing,
keyId: keyId,
indexerPath: indexerPath,
blobPath: blobPath,
packBlobs: packBlobs,
searchOwner: blob.SHA1FromString(armoredPublicKey),
shareHandlerPath: shareHandlerPath,
flickr: flickr,
memoryIndex: memoryIndex,
}
prefixes := genLowLevelPrefixes(prefixesParams, ownerName)
var cacheDir string
if nolocaldisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket
// See http://code.google.com/p/camlistore/issues/detail?id=85
cacheDir = filepath.Join(tempDir(), "camli-cache")
} else {
cacheDir = filepath.Join(blobPath, "cache")
}
if !noMkdir {
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
}
}
published := []interface{}{}
if len(publish) > 0 {
if !runIndex {
return nil, fmt.Errorf("publishing requires an index")
}
published, err = addPublishedConfig(prefixes, publish, sourceRoot)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
if runIndex {
addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot)
}
if mysql != "" {
addMySQLConfig(prefixes, dbname, mysql)
}
if postgres != "" {
addPostgresConfig(prefixes, dbname, postgres)
}
if mongo != "" {
addMongoConfig(prefixes, dbname, mongo)
}
if sqliteFile != "" {
addSQLiteConfig(prefixes, sqliteFile)
}
if kvFile != "" {
addKVConfig(prefixes, kvFile)
}
if s3 != "" {
if err := addS3Config(prefixesParams, prefixes, s3); err != nil {
return nil, err
}
}
if googledrive != "" {
if err := addGoogleDriveConfig(prefixes, googledrive); err != nil {
return nil, err
}
}
if googlecloudstorage != "" {
if err := addGoogleCloudStorageConfig(prefixes, googlecloudstorage); err != nil {
return nil, err
}
}
if swift != "" {
if err := addSwiftConfig(prefixesParams, prefixes, swift); err != nil {
return nil, err
}
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
configPath: conf.configPath,
}
return lowLevelConf, nil
}
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
var (
baseURL = conf.OptionalString("baseURL", "")
listen = conf.OptionalString("listen", "")
auth = conf.RequiredString("auth")
keyId = conf.RequiredString("identity")
secretRing = conf.RequiredString("identitySecretRing")
blobPath = conf.RequiredString("blobPath")
tlsOn = conf.OptionalBool("https", false)
tlsCert = conf.OptionalString("HTTPSCertFile", "")
tlsKey = conf.OptionalString("HTTPSKeyFile", "")
dbname = conf.OptionalString("dbname", "")
mysql = conf.OptionalString("mysql", "")
postgres = conf.OptionalString("postgres", "")
mongo = conf.OptionalString("mongo", "")
_ = conf.OptionalList("replicateTo")
s3 = conf.OptionalString("s3", "")
publish = conf.OptionalObject("publish")
)
if err := conf.Validate(); err != nil {
return nil, err
}
obj := jsonconfig.Obj{}
if tlsOn {
if (tlsCert != "") != (tlsKey != "") {
return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
}
if tlsCert != "" {
obj["TLSCertFile"] = tlsCert
obj["TLSKeyFile"] = tlsKey
} else {
obj["TLSCertFile"] = "config/selfgen_cert.pem"
obj["TLSKeyFile"] = "config/selfgen_key.pem"
}
}
if baseURL != "" {
if strings.HasSuffix(baseURL, "/") {
baseURL = baseURL[:len(baseURL)-1]
}
obj["baseURL"] = baseURL
}
if listen != "" {
obj["listen"] = listen
}
obj["https"] = tlsOn
obj["auth"] = auth
if dbname == "" {
username := os.Getenv("USER")
if username == "" {
return nil, fmt.Errorf("USER env var not set; needed to define dbname")
}
dbname = "camli" + username
}
var indexerPath string
switch {
case mongo != "" && mysql != "" || mongo != "" && postgres != "" || mysql != "" && postgres != "":
return nil, fmt.Errorf("You can only pick one of the db engines (mongo, mysql, postgres).")
case mysql != "":
indexerPath = "/index-mysql/"
case postgres != "":
indexerPath = "/index-postgres/"
case mongo != "":
indexerPath = "/index-mongo/"
default:
indexerPath = "/index-mem/"
}
entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
prefixesParams := &configPrefixesParams{
secretRing: secretRing,
keyId: keyId,
indexerPath: indexerPath,
blobPath: blobPath,
searchOwner: blobref.SHA1FromString(armoredPublicKey),
}
prefixes := genLowLevelPrefixes(prefixesParams)
cacheDir := filepath.Join(blobPath, "/cache")
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs dir %s: %v", cacheDir, err)
}
published := []interface{}{}
if publish != nil {
published, err = addPublishedConfig(prefixes, publish)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
addUIConfig(prefixes, "/ui/", published)
if mysql != "" {
addMySQLConfig(prefixes, dbname, mysql)
}
if postgres != "" {
addPostgresConfig(prefixes, dbname, postgres)
}
if mongo != "" {
addMongoConfig(prefixes, dbname, mongo)
}
if s3 != "" {
if err := addS3Config(prefixes, s3); err != nil {
return nil, err
}
}
if indexerPath == "/index-mem/" {
addMemindexConfig(prefixes)
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
Obj: obj,
configPath: conf.configPath,
}
return lowLevelConf, nil
}
func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
var (
baseUrl = conf.RequiredString("listen")
auth = conf.RequiredString("auth")
keyId = conf.RequiredString("identity")
secretRing = conf.RequiredString("identitySecretRing")
blobPath = conf.RequiredString("blobPath")
tlsOn = conf.OptionalBool("TLS", false)
dbname = conf.OptionalString("dbname", "")
mysql = conf.OptionalString("mysql", "")
mongo = conf.OptionalString("mongo", "")
_ = conf.OptionalList("replicateTo")
_ = conf.OptionalString("s3", "")
publish = conf.OptionalObject("publish")
)
if err := conf.Validate(); err != nil {
return nil, err
}
obj := jsonconfig.Obj{}
scheme := "http"
if tlsOn {
scheme = "https"
obj["TLSCertFile"] = "config/selfgen_cert.pem"
obj["TLSKeyFile"] = "config/selfgen_key.pem"
}
obj["baseURL"] = scheme + "://" + baseUrl
obj["https"] = tlsOn
obj["auth"] = auth
if dbname == "" {
username := os.Getenv("USER")
if username == "" {
return nil, fmt.Errorf("USER env var not set; needed to define dbname")
}
dbname = "camli" + username
}
var indexerPath string
switch {
case mongo != "" && mysql != "":
return nil, fmt.Errorf("Cannot have both mysql and mongo in config, pick one")
case mysql != "":
indexerPath = "/index-mysql/"
case mongo != "":
indexerPath = "/index-mongo/"
default:
indexerPath = "/index-mem/"
}
entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
if err != nil {
return nil, err
}
armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
if err != nil {
return nil, err
}
prefixesParams := &configPrefixesParams{
secretRing: secretRing,
keyId: keyId,
indexerPath: indexerPath,
blobPath: blobPath,
searchOwner: blobref.SHA1FromString(armoredPublicKey),
}
prefixes := genLowLevelPrefixes(prefixesParams)
cacheDir := filepath.Join(blobPath, "/cache")
if err := os.MkdirAll(cacheDir, 0700); err != nil {
return nil, fmt.Errorf("Could not create blobs dir %s: %v", cacheDir, err)
}
published := []interface{}{}
if publish != nil {
published, err = addPublishedConfig(&prefixes, publish)
if err != nil {
return nil, fmt.Errorf("Could not generate config for published: %v", err)
}
}
addUIConfig(&prefixes, "/ui/", published)
if mysql != "" {
addMysqlConfig(&prefixes, dbname, mysql)
}
if mongo != "" {
addMongoConfig(&prefixes, dbname, mongo)
}
if indexerPath == "/index-mem/" {
addMemindexConfig(&prefixes)
}
obj["prefixes"] = (map[string]interface{})(prefixes)
lowLevelConf = &Config{
jsonconfig.Obj: obj,
configPath: conf.configPath,
}
return lowLevelConf, nil
}
