Example #1
0
func (c *Client) initSignerPublicKeyBlobref() {
	if c.paramsOnly {
		log.Print("client: paramsOnly set; cannot get public key from config or env vars.")
		return
	}
	keyId := os.Getenv("CAMLI_KEYID")
	if keyId == "" {
		configOnce.Do(parseConfig)
		keyId = config.Identity
		if keyId == "" {
			log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"camput init\"?", osutil.UserClientConfigPath())
		}
	}
	keyRing := c.SecretRingFile()
	if !fileExists(keyRing) {
		log.Fatalf("Could not find keyId %q, because secret ring file %q does not exist.", keyId, keyRing)
	}
	entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
	if err != nil {
		log.Fatalf("Couldn't find keyId %q in secret ring %v: %v", keyId, keyRing, err)
	}
	armored, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		log.Fatalf("Error serializing public key: %v", err)
	}

	c.signerPublicKeyRef = blob.SHA1FromString(armored)
	c.publicKeyArmored = armored
}
Example #2
0
func (e *Env) SetCamdevVars(altkey bool) {
	e.Set("CAMLI_CONFIG_DIR", filepath.Join("config", "dev-client-dir"))
	e.Set("CAMLI_AUTH", "userpass:camlistore:pass3179")
	e.Set("CAMLI_DEV_KEYBLOBS", filepath.FromSlash("config/dev-client-dir/keyblobs"))

	secring := defaultSecring
	identity := defaultIdentity

	if altkey {
		secring = filepath.FromSlash("pkg/jsonsign/testdata/password-foo-secring.gpg")
		identity = "C7C3E176"
		println("**\n** Note: password is \"foo\"\n**\n")
	} else {
		if *flagSecretRing != "" {
			secring = *flagSecretRing
		}
		if *flagIdentity != "" {
			identity = *flagIdentity
		}
	}

	entity, err := jsonsign.EntityFromSecring(identity, secring)
	if err != nil {
		panic(err)
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		panic(err)
	}
	pubKeyRef := blob.SHA1FromString(armoredPublicKey)

	e.Set("CAMLI_SECRET_RING", secring)
	e.Set("CAMLI_KEYID", identity)
	e.Set("CAMLI_PUBKEY_BLOBREF", pubKeyRef.String())
}
Example #3
0
func TestSigner(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping in short mode")
	}
	ent, err := jsonsign.NewEntity()
	if err != nil {
		t.Fatal(err)
	}
	armorPub, err := jsonsign.ArmoredPublicKey(ent)
	if err != nil {
		t.Fatal(err)
	}
	pubRef := blob.SHA1FromString(armorPub)
	sig, err := NewSigner(pubRef, strings.NewReader(armorPub), ent)
	if err != nil {
		t.Fatalf("NewSigner: %v", err)
	}
	pn, err := NewUnsignedPermanode().Sign(sig)
	if err != nil {
		t.Fatalf("NewPermanode: %v", err)
	}
	if !strings.Contains(pn, `,"camliSig":"`) {
		t.Errorf("Permanode doesn't look signed: %v", pn)
	}
}
Example #4
0
func getSignerPublicKeyBlobref() (signerRef blob.Ref, armored string, ok bool) {
	configOnce.Do(parseConfig)
	key := "keyId"
	keyId, ok := config[key].(string)
	if !ok {
		log.Printf("No key %q in JSON configuration file %q; have you run \"camput init\"?", key, osutil.UserClientConfigPath())
		return
	}
	keyRing, hasKeyRing := config["secretRing"].(string)
	if !hasKeyRing {
		if fn := osutil.IdentitySecretRing(); fileExists(fn) {
			keyRing = fn
		} else if fn := jsonsign.DefaultSecRingPath(); fileExists(fn) {
			keyRing = fn
		} else {
			log.Printf("Couldn't find keyId %q; no 'secretRing' specified in config file, and no standard secret ring files exist.")
			return
		}
	}
	entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
	if err != nil {
		log.Printf("Couldn't find keyId %q in secret ring: %v", keyId, err)
		return
	}
	armored, err = jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		log.Printf("Error serializing public key: %v", err)
		return
	}

	// TODO(mpl): integrate with getSelfPubKeyDir if possible.
	selfPubKeyDir, ok := config["selfPubKeyDir"].(string)
	if !ok {
		selfPubKeyDir = osutil.KeyBlobsDir()
		log.Printf("No 'selfPubKeyDir' defined in %q, defaulting to %v", osutil.UserClientConfigPath(), selfPubKeyDir)
	}
	fi, err := os.Stat(selfPubKeyDir)
	if err != nil || !fi.IsDir() {
		log.Printf("selfPubKeyDir of %q doesn't exist or not a directory", selfPubKeyDir)
		return
	}

	br := blob.SHA1FromString(armored)

	pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
	fi, err = os.Stat(pubFile)
	if err != nil {
		err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
		if err != nil {
			log.Printf("Error writing public key to %q: %v", pubFile, err)
			return
		}
	}

	return br, armored, true
}
Example #5
0
// TODO: move to config package?
func SignerPublicKeyBlobref() *blobref.BlobRef {
	configOnce.Do(parseConfig)
	key := "keyId"
	keyId, ok := config[key].(string)
	if !ok {
		log.Printf("No key %q in JSON configuration file %q; have you run \"camput init\"?", key, ConfigFilePath())
		return nil
	}
	keyRing, hasKeyRing := config["secretRing"].(string)
	if !hasKeyRing {
		if fn := osutil.IdentitySecretRing(); fileExists(fn) {
			keyRing = fn
		} else if fn := jsonsign.DefaultSecRingPath(); fileExists(fn) {
			keyRing = fn
		} else {
			log.Printf("Couldn't find keyId %q; no 'secretRing' specified in config file, and no standard secret ring files exist.")
			return nil
		}
	}
	entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
	if err != nil {
		log.Printf("Couldn't find keyId %q in secret ring: %v", keyId, err)
		return nil
	}
	armored, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		log.Printf("Error serializing public key: %v", err)
		return nil
	}

	selfPubKeyDir, ok := config["selfPubKeyDir"].(string)
	if !ok {
		log.Printf("No 'selfPubKeyDir' defined in %q", ConfigFilePath())
		return nil
	}
	fi, err := os.Stat(selfPubKeyDir)
	if err != nil || !fi.IsDir() {
		log.Printf("selfPubKeyDir of %q doesn't exist or not a directory", selfPubKeyDir)
		return nil
	}

	br := blobref.SHA1FromString(armored)

	pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
	log.Printf("key file: %q", pubFile)
	fi, err = os.Stat(pubFile)
	if err != nil {
		err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
		if err != nil {
			log.Printf("Error writing public key to %q: %v", pubFile, err)
			return nil
		}
	}

	return br
}
Example #6
0
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
	pubKeyDestPrefix := conf.OptionalString("publicKeyDest", "")

	// either a short form ("26F5ABDA") or one the longer forms.
	keyId := conf.RequiredString("keyId")

	h := &Handler{
		secretRing: conf.OptionalString("secretRing", ""),
	}
	var err error
	if err = conf.Validate(); err != nil {
		return nil, err
	}

	h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
	if err != nil {
		return nil, err
	}

	armoredPublicKey, err := jsonsign.ArmoredPublicKey(h.entity)

	ms := new(blobref.MemoryStore)
	h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, armoredPublicKey)
	if err != nil {
		return nil, err
	}
	h.pubKeyFetcher = ms

	if pubKeyDestPrefix != "" {
		sto, err := ld.GetStorage(pubKeyDestPrefix)
		if err != nil {
			return nil, err
		}
		h.pubKeyDest = sto
		if sto != nil {
			if ctxReq, ok := ld.GetRequestContext(); ok {
				if w, ok := sto.(blobserver.ContextWrapper); ok {
					sto = w.WrapContext(ctxReq)
				}
			}
			err := h.uploadPublicKey(sto, armoredPublicKey)
			if err != nil {
				return nil, fmt.Errorf("Error seeding self public key in storage: %v", err)
			}
		}
	}
	h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
	h.pubKeyHandler = &gethandler.Handler{
		Fetcher:           ms,
		AllowGlobalAccess: true, // just public keys
	}

	return h, nil
}
Example #7
0
func (c *initCmd) getPublicKeyArmored() ([]byte, error) {
	entity, err := jsonsign.EntityFromSecring(c.keyId, c.secretRing)
	if err != nil {
		return nil, fmt.Errorf("Could not find keyId %v in ring %v: %v", c.keyId, c.secretRing, err)
	}
	pubArmor, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, fmt.Errorf("failed to export armored public key ID %q from %v: %v", c.keyId, c.secretRing, err)
	}
	return []byte(pubArmor), nil
}
Example #8
0
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
	pubKeyDestPrefix := conf.OptionalString("publicKeyDest", "")

	// either a short form ("26F5ABDA") or one the longer forms.
	keyId := conf.RequiredString("keyId")

	h := &Handler{
		secretRing: conf.OptionalString("secretRing", ""),
	}
	var err error
	if err = conf.Validate(); err != nil {
		return nil, err
	}

	h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
	if err != nil {
		return nil, err
	}

	armoredPublicKey, err := jsonsign.ArmoredPublicKey(h.entity)

	ms := new(blob.MemoryStore)
	h.pubKeyBlobRef, err = ms.AddBlob(crypto.SHA1, armoredPublicKey)
	if err != nil {
		return nil, err
	}
	h.pubKeyFetcher = ms

	if pubKeyDestPrefix != "" {
		sto, err := ld.GetStorage(pubKeyDestPrefix)
		if err != nil {
			return nil, err
		}
		h.pubKeyDest = sto
		if sto != nil {
			err := h.uploadPublicKey(sto, armoredPublicKey)
			if err != nil {
				return nil, fmt.Errorf("Error seeding self public key in storage: %v", err)
			}
		}
	}
	h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
	h.pubKeyHandler = &gethandler.Handler{
		Fetcher: ms,
	}

	h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(armoredPublicKey), h.entity)
	if err != nil {
		return nil, err
	}

	return h, nil
}
Example #9
0
// Error is errNoOwner if no identity configured
func (b *lowBuilder) searchOwner() (br blob.Ref, err error) {
	if b.high.Identity == "" {
		return br, errNoOwner
	}
	entity, err := jsonsign.EntityFromSecring(b.high.Identity, b.high.IdentitySecretRing)
	if err != nil {
		return br, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return br, err
	}
	return blob.SHA1FromString(armoredPublicKey), nil
}
Example #10
0
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
	var (
		// either a short form ("26F5ABDA") or one the longer forms.
		keyId = conf.RequiredString("keyId")

		pubKeyDestPrefix = conf.OptionalString("publicKeyDest", "")
		secretRing       = conf.OptionalString("secretRing", "")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	h := &Handler{
		secretRing: secretRing,
	}

	var err error
	h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath())
	if err != nil {
		return nil, err
	}

	h.pubKey, err = jsonsign.ArmoredPublicKey(h.entity)

	ms := &memory.Storage{}
	h.pubKeyBlobRef = blob.SHA1FromString(h.pubKey)
	if _, err := ms.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(h.pubKey)); err != nil {
		return nil, fmt.Errorf("could not store pub key blob: %v", err)
	}
	h.pubKeyFetcher = ms

	if pubKeyDestPrefix != "" {
		sto, err := ld.GetStorage(pubKeyDestPrefix)
		if err != nil {
			return nil, err
		}
		h.pubKeyDest = sto
	}
	h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String()
	h.pubKeyHandler = &gethandler.Handler{
		Fetcher: ms,
	}

	h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(h.pubKey), h.entity)
	if err != nil {
		return nil, err
	}

	return h, nil
}
Example #11
0
func setCamdevVarsFor(e *Env, altkey bool) {
	var setenv func(string, string) error
	if e != nil {
		setenv = func(k, v string) error { e.Set(k, v); return nil }
	} else {
		setenv = os.Setenv
	}

	setenv("CAMLI_AUTH", "userpass:camlistore:pass3179")
	// env values for clients. server will overwrite them anyway in its setEnvVars.
	root, err := rootInTmpDir()
	if err != nil {
		log.Fatal(err)
	}
	setenv("CAMLI_CACHE_DIR", filepath.Join(root, "client", "cache"))
	setenv("CAMLI_CONFIG_DIR", filepath.Join("config", "dev-client-dir"))

	secring := defaultSecring
	identity := defaultIdentity

	if altkey {
		secring = filepath.FromSlash("pkg/jsonsign/testdata/password-foo-secring.gpg")
		identity = "C7C3E176"
		println("**\n** Note: password is \"foo\"\n**\n")
	} else {
		if *flagSecretRing != "" {
			secring = *flagSecretRing
		}
		if *flagIdentity != "" {
			identity = *flagIdentity
		}
	}

	entity, err := jsonsign.EntityFromSecring(identity, secring)
	if err != nil {
		panic(err)
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		panic(err)
	}
	pubKeyRef := blob.SHA1FromString(armoredPublicKey)

	setenv("CAMLI_SECRET_RING", secring)
	setenv("CAMLI_KEYID", identity)
	setenv("CAMLI_PUBKEY_BLOBREF", pubKeyRef.String())
	setenv("CAMLI_KV_VERIFY", "true")
}
Example #12
0
func (c *initCmd) getPublicKeyArmoredFromFile(secretRingFileName, keyId string) (b []byte, err error) {
	entity, err := jsonsign.EntityFromSecring(keyId, secretRingFileName)
	if err == nil {
		pubArmor, err := jsonsign.ArmoredPublicKey(entity)
		if err == nil {
			return []byte(pubArmor), nil
		}
	}
	b, err = exec.Command("gpg", "--export", "--armor", keyId).Output()
	if err != nil {
		return nil, fmt.Errorf("Error running gpg to export public key %q: %v", keyId, err)
	}
	if len(b) == 0 {
		return nil, fmt.Errorf("gpg export of public key %q was empty.", keyId)
	}
	return b, nil
}
Example #13
0
func (c *Client) initSignerPublicKeyBlobref() {
	configOnce.Do(parseConfig)
	keyId := config.identity
	if keyId == "" {
		log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"camput init\"?", osutil.UserClientConfigPath())
	}
	keyRing := c.SecretRingFile()
	if !fileExists(keyRing) {
		log.Fatalf("Could not find keyId %q, because secret ring file %q does not exist.", keyId, keyRing)
	}
	entity, err := jsonsign.EntityFromSecring(keyId, keyRing)
	if err != nil {
		log.Fatalf("Couldn't find keyId %q in secret ring %v: %v", keyId, keyRing, err)
	}
	armored, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		log.Fatalf("Error serializing public key: %v", err)
	}

	// TODO(mpl): completely get rid of it if possible
	// http://camlistore.org/issue/377
	selfPubKeyDir := osutil.KeyBlobsDir()
	fi, err := os.Stat(selfPubKeyDir)
	if err != nil || !fi.IsDir() {
		log.Fatalf("selfPubKeyDir as %q doesn't exist or not a directory", selfPubKeyDir)
	}

	br := blob.SHA1FromString(armored)
	pubFile := filepath.Join(selfPubKeyDir, br.String()+".camli")
	fi, err = os.Stat(pubFile)
	if err != nil {
		if !os.IsNotExist(err) {
			log.Fatalf("Could not stat %q: %v", pubFile, err)
		}
		err = ioutil.WriteFile(pubFile, []byte(armored), 0644)
		if err != nil {
			log.Fatalf("Error writing public key to %q: %v", pubFile, err)
		}
	}
	c.signerPublicKeyRef = br
	c.publicKeyArmored = armored
}
Example #14
0
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
	var (
		baseURL    = conf.OptionalString("baseURL", "")
		listen     = conf.OptionalString("listen", "")
		auth       = conf.RequiredString("auth")
		keyId      = conf.RequiredString("identity")
		secretRing = conf.RequiredString("identitySecretRing")
		tlsOn      = conf.OptionalBool("https", false)
		tlsCert    = conf.OptionalString("HTTPSCertFile", "")
		tlsKey     = conf.OptionalString("HTTPSKeyFile", "")

		// Blob storage options
		blobPath     = conf.OptionalString("blobPath", "")
		s3           = conf.OptionalString("s3", "")           // "access_key_id:secret_access_key:bucket"
		shareHandler = conf.OptionalBool("shareHandler", true) // enable the share handler

		// Index options
		runIndex   = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc.
		dbname     = conf.OptionalString("dbname", "")   // for mysql, postgres, mongo
		mysql      = conf.OptionalString("mysql", "")
		postgres   = conf.OptionalString("postgres", "")
		memIndex   = conf.OptionalBool("memIndex", false)
		mongo      = conf.OptionalString("mongo", "")
		sqliteFile = conf.OptionalString("sqlite", "")

		_       = conf.OptionalList("replicateTo")
		publish = conf.OptionalObject("publish")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	obj := jsonconfig.Obj{}
	if tlsOn {
		if (tlsCert != "") != (tlsKey != "") {
			return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
		}
		if tlsCert != "" {
			obj["TLSCertFile"] = tlsCert
			obj["TLSKeyFile"] = tlsKey
		} else {
			obj["TLSCertFile"] = DefaultTLSCert
			obj["TLSKeyFile"] = DefaultTLSKey
		}
	}

	if baseURL != "" {
		if strings.HasSuffix(baseURL, "/") {
			baseURL = baseURL[:len(baseURL)-1]
		}
		obj["baseURL"] = baseURL
	}
	if listen != "" {
		obj["listen"] = listen
	}
	obj["https"] = tlsOn
	obj["auth"] = auth

	if dbname == "" {
		username := os.Getenv("USER")
		if username == "" {
			return nil, fmt.Errorf("USER env var not set; needed to define dbname")
		}
		dbname = "camli" + username
	}

	var indexerPath string
	numIndexers := numSet(mongo, mysql, postgres, sqliteFile, memIndex)
	switch {
	case runIndex && numIndexers == 0:
		return nil, fmt.Errorf("Unless wantIndex is set to false, you must specify an index option (mongo, mysql, postgres, sqlite, memIndex).")
	case runIndex && numIndexers != 1:
		return nil, fmt.Errorf("With wantIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite, memIndex).")
	case !runIndex && numIndexers != 0:
		return nil, fmt.Errorf("With wantIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite, memIndex.")
	case mysql != "":
		indexerPath = "/index-mysql/"
	case postgres != "":
		indexerPath = "/index-postgres/"
	case mongo != "":
		indexerPath = "/index-mongo/"
	case sqliteFile != "":
		indexerPath = "/index-sqlite/"
	case memIndex:
		indexerPath = "/index-mem/"
	}

	entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
	if err != nil {
		return nil, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, err
	}

	nolocaldisk := blobPath == ""
	if nolocaldisk && s3 == "" {
		return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 configured for a blobserver.")
	}

	prefixesParams := &configPrefixesParams{
		secretRing:   secretRing,
		keyId:        keyId,
		indexerPath:  indexerPath,
		blobPath:     blobPath,
		searchOwner:  blobref.SHA1FromString(armoredPublicKey),
		shareHandler: shareHandler,
	}

	prefixes := genLowLevelPrefixes(prefixesParams)
	var cacheDir string
	if nolocaldisk {
		// Whether camlistored is run from EC2 or not, we use
		// a temp dir as the cache when primary storage is S3.
		// TODO(mpl): s3CacheBucket
		// See http://code.google.com/p/camlistore/issues/detail?id=85
		cacheDir = filepath.Join(tempDir(), "camli-cache")
	} else {
		cacheDir = filepath.Join(blobPath, "/cache")
	}
	if err := os.MkdirAll(cacheDir, 0700); err != nil {
		return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
	}

	published := []interface{}{}
	if len(publish) > 0 {
		if !runIndex {
			return nil, fmt.Errorf("publishing requires an index")
		}
		published, err = addPublishedConfig(prefixes, publish)
		if err != nil {
			return nil, fmt.Errorf("Could not generate config for published: %v", err)
		}
	}

	if runIndex {
		addUIConfig(prefixes, "/ui/", published)
	}

	if mysql != "" {
		addMySQLConfig(prefixes, dbname, mysql)
	}
	if postgres != "" {
		addPostgresConfig(prefixes, dbname, postgres)
	}
	if mongo != "" {
		addMongoConfig(prefixes, dbname, mongo)
	}
	if sqliteFile != "" {
		addSQLiteConfig(prefixes, sqliteFile)
	}
	if s3 != "" {
		if err := addS3Config(prefixes, s3); err != nil {
			return nil, err
		}
	}
	if indexerPath == "/index-mem/" {
		addMemindexConfig(prefixes)
	}

	obj["prefixes"] = (map[string]interface{})(prefixes)

	lowLevelConf = &Config{
		Obj:        obj,
		configPath: conf.configPath,
	}
	return lowLevelConf, nil
}
Example #15
0
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
	obj := jsonconfig.Obj{}
	if conf.HTTPS {
		if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") {
			return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)")
		}
		if conf.HTTPSCert != "" {
			obj["httpsCert"] = conf.HTTPSCert
			obj["httpsKey"] = conf.HTTPSKey
		} else {
			obj["httpsCert"] = osutil.DefaultTLSCert()
			obj["httpsKey"] = osutil.DefaultTLSKey()
		}
	}

	if conf.BaseURL != "" {
		u, err := url.Parse(conf.BaseURL)
		if err != nil {
			return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", conf.BaseURL, err)
		}
		if u.Path != "" && u.Path != "/" {
			return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
		}
		u.Path = ""
		obj["baseURL"] = u.String()
	}
	if conf.Listen != "" {
		obj["listen"] = conf.Listen
	}
	obj["https"] = conf.HTTPS
	obj["auth"] = conf.Auth

	username := ""
	if conf.DBName == "" {
		username = osutil.Username()
		if username == "" {
			return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
		}
		conf.DBName = "camli" + username
	}

	var indexerPath string
	numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile)
	runIndex := conf.RunIndex.Get()
	switch {
	case runIndex && numIndexers == 0:
		return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
	case runIndex && numIndexers != 1:
		return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
	case !runIndex && numIndexers != 0:
		return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
	case conf.MySQL != "":
		indexerPath = "/index-mysql/"
	case conf.PostgreSQL != "":
		indexerPath = "/index-postgres/"
	case conf.Mongo != "":
		indexerPath = "/index-mongo/"
	case conf.SQLite != "":
		indexerPath = "/index-sqlite/"
	case conf.KVFile != "":
		indexerPath = "/index-kv/"
	}

	entity, err := jsonsign.EntityFromSecring(conf.Identity, conf.IdentitySecretRing)
	if err != nil {
		return nil, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, err
	}

	nolocaldisk := conf.BlobPath == ""
	if nolocaldisk {
		if conf.S3 == "" && conf.GoogleCloudStorage == "" {
			return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
		}
		if conf.S3 != "" && conf.GoogleCloudStorage != "" {
			return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
		}
	}

	if conf.ShareHandler && conf.ShareHandlerPath == "" {
		conf.ShareHandlerPath = "/share/"
	}

	prefixesParams := &configPrefixesParams{
		secretRing:       conf.IdentitySecretRing,
		keyId:            conf.Identity,
		indexerPath:      indexerPath,
		blobPath:         conf.BlobPath,
		packBlobs:        conf.PackBlobs,
		searchOwner:      blob.SHA1FromString(armoredPublicKey),
		shareHandlerPath: conf.ShareHandlerPath,
		flickr:           conf.Flickr,
		memoryIndex:      conf.MemoryIndex.Get(),
	}

	prefixes := genLowLevelPrefixes(prefixesParams, conf.OwnerName)
	var cacheDir string
	if nolocaldisk {
		// Whether camlistored is run from EC2 or not, we use
		// a temp dir as the cache when primary storage is S3.
		// TODO(mpl): s3CacheBucket
		// See http://code.google.com/p/camlistore/issues/detail?id=85
		cacheDir = filepath.Join(tempDir(), "camli-cache")
	} else {
		cacheDir = filepath.Join(conf.BlobPath, "cache")
	}
	if !noMkdir {
		if err := os.MkdirAll(cacheDir, 0700); err != nil {
			return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
		}
	}

	published := []interface{}{}
	if len(conf.Publish) > 0 {
		if !runIndex {
			return nil, fmt.Errorf("publishing requires an index")
		}
		published, err = addPublishedConfig(prefixes, conf.Publish, conf.SourceRoot)
		if err != nil {
			return nil, fmt.Errorf("Could not generate config for published: %v", err)
		}
	}

	if runIndex {
		addUIConfig(prefixesParams, prefixes, "/ui/", published, conf.SourceRoot)
	}

	if conf.MySQL != "" {
		addMySQLConfig(prefixes, conf.DBName, conf.MySQL)
	}
	if conf.PostgreSQL != "" {
		addPostgresConfig(prefixes, conf.DBName, conf.PostgreSQL)
	}
	if conf.Mongo != "" {
		addMongoConfig(prefixes, conf.DBName, conf.Mongo)
	}
	if conf.SQLite != "" {
		addSQLiteConfig(prefixes, conf.SQLite)
	}
	if conf.KVFile != "" {
		addKVConfig(prefixes, conf.KVFile)
	}
	if conf.S3 != "" {
		if err := addS3Config(prefixesParams, prefixes, conf.S3); err != nil {
			return nil, err
		}
	}
	if conf.GoogleDrive != "" {
		if err := addGoogleDriveConfig(prefixes, conf.GoogleDrive); err != nil {
			return nil, err
		}
	}
	if conf.GoogleCloudStorage != "" {
		if err := addGoogleCloudStorageConfig(prefixes, conf.GoogleCloudStorage); err != nil {
			return nil, err
		}
	}

	obj["prefixes"] = (map[string]interface{})(prefixes)

	lowLevelConf = &Config{
		Obj: obj,
	}
	return lowLevelConf, nil
}
Example #16
0
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
	var (
		baseURL    = conf.OptionalString("baseURL", "")
		listen     = conf.OptionalString("listen", "")
		auth       = conf.RequiredString("auth")
		keyId      = conf.RequiredString("identity")
		secretRing = conf.RequiredString("identitySecretRing")
		tlsOn      = conf.OptionalBool("https", false)
		tlsCert    = conf.OptionalString("HTTPSCertFile", "")
		tlsKey     = conf.OptionalString("HTTPSKeyFile", "")

		// Blob storage options
		blobPath           = conf.OptionalString("blobPath", "")
		packBlobs          = conf.OptionalBool("packBlobs", false)         // use diskpacked instead of the default filestorage
		s3                 = conf.OptionalString("s3", "")                 // "access_key_id:secret_access_key:bucket[:hostname]"
		googlecloudstorage = conf.OptionalString("googlecloudstorage", "") // "clientId:clientSecret:refreshToken:bucket"
		googledrive        = conf.OptionalString("googledrive", "")        // "clientId:clientSecret:refreshToken:parentId"
		swift              = conf.OptionalString("swift", "")              // "tenant:secret:container:auth_url"
		// Enable the share handler. If true, and shareHandlerPath is empty,
		// then shareHandlerPath defaults to "/share/".
		shareHandler = conf.OptionalBool("shareHandler", false)
		// URL prefix for the share handler. If set, overrides shareHandler.
		shareHandlerPath = conf.OptionalString("shareHandlerPath", "")

		// Index options
		memoryIndex = conf.OptionalBool("memoryIndex", true) // copy disk-based index to memory on start-up
		runIndex    = conf.OptionalBool("runIndex", true)    // if false: no search, no UI, etc.
		dbname      = conf.OptionalString("dbname", "")      // for mysql, postgres, mongo
		mysql       = conf.OptionalString("mysql", "")
		postgres    = conf.OptionalString("postgres", "")
		mongo       = conf.OptionalString("mongo", "")
		sqliteFile  = conf.OptionalString("sqlite", "")
		kvFile      = conf.OptionalString("kvIndexFile", "")

		// Importer options
		flickr = conf.OptionalString("flickr", "")

		_       = conf.OptionalList("replicateTo")
		publish = conf.OptionalObject("publish")
		// alternative source tree, to override the embedded ui and/or closure resources.
		// If non empty, the ui files will be expected at
		// sourceRoot + "/server/camlistored/ui" and the closure library at
		// sourceRoot + "/third_party/closure/lib"
		// Also used by the publish handler.
		sourceRoot = conf.OptionalString("sourceRoot", "")

		ownerName = conf.OptionalString("ownerName", "")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	obj := jsonconfig.Obj{}
	if tlsOn {
		if (tlsCert != "") != (tlsKey != "") {
			return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
		}
		if tlsCert != "" {
			obj["TLSCertFile"] = tlsCert
			obj["TLSKeyFile"] = tlsKey
		} else {
			obj["TLSCertFile"] = osutil.DefaultTLSCert()
			obj["TLSKeyFile"] = osutil.DefaultTLSKey()
		}
	}

	if baseURL != "" {
		u, err := url.Parse(baseURL)
		if err != nil {
			return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", baseURL, err)
		}
		if u.Path != "" && u.Path != "/" {
			return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port.")
		}
		u.Path = ""
		obj["baseURL"] = u.String()
	}
	if listen != "" {
		obj["listen"] = listen
	}
	obj["https"] = tlsOn
	obj["auth"] = auth

	username := ""
	if dbname == "" {
		username = osutil.Username()
		if username == "" {
			return nil, fmt.Errorf("USER (USERNAME on windows) env var not set; needed to define dbname")
		}
		dbname = "camli" + username
	}

	var indexerPath string
	numIndexers := numSet(mongo, mysql, postgres, sqliteFile, kvFile)
	switch {
	case runIndex && numIndexers == 0:
		return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
	case runIndex && numIndexers != 1:
		return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
	case !runIndex && numIndexers != 0:
		return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
	case mysql != "":
		indexerPath = "/index-mysql/"
	case postgres != "":
		indexerPath = "/index-postgres/"
	case mongo != "":
		indexerPath = "/index-mongo/"
	case sqliteFile != "":
		indexerPath = "/index-sqlite/"
	case kvFile != "":
		indexerPath = "/index-kv/"
	}

	entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
	if err != nil {
		return nil, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, err
	}

	nolocaldisk := blobPath == ""
	if nolocaldisk {
		if s3 == "" && googlecloudstorage == "" {
			return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
		}
		if s3 != "" && googlecloudstorage != "" {
			return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
		}
	}

	if shareHandler && shareHandlerPath == "" {
		shareHandlerPath = "/share/"
	}

	prefixesParams := &configPrefixesParams{
		secretRing:       secretRing,
		keyId:            keyId,
		indexerPath:      indexerPath,
		blobPath:         blobPath,
		packBlobs:        packBlobs,
		searchOwner:      blob.SHA1FromString(armoredPublicKey),
		shareHandlerPath: shareHandlerPath,
		flickr:           flickr,
		memoryIndex:      memoryIndex,
	}

	prefixes := genLowLevelPrefixes(prefixesParams, ownerName)
	var cacheDir string
	if nolocaldisk {
		// Whether camlistored is run from EC2 or not, we use
		// a temp dir as the cache when primary storage is S3.
		// TODO(mpl): s3CacheBucket
		// See http://code.google.com/p/camlistore/issues/detail?id=85
		cacheDir = filepath.Join(tempDir(), "camli-cache")
	} else {
		cacheDir = filepath.Join(blobPath, "cache")
	}
	if !noMkdir {
		if err := os.MkdirAll(cacheDir, 0700); err != nil {
			return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
		}
	}

	published := []interface{}{}
	if len(publish) > 0 {
		if !runIndex {
			return nil, fmt.Errorf("publishing requires an index")
		}
		published, err = addPublishedConfig(prefixes, publish, sourceRoot)
		if err != nil {
			return nil, fmt.Errorf("Could not generate config for published: %v", err)
		}
	}

	if runIndex {
		addUIConfig(prefixesParams, prefixes, "/ui/", published, sourceRoot)
	}

	if mysql != "" {
		addMySQLConfig(prefixes, dbname, mysql)
	}
	if postgres != "" {
		addPostgresConfig(prefixes, dbname, postgres)
	}
	if mongo != "" {
		addMongoConfig(prefixes, dbname, mongo)
	}
	if sqliteFile != "" {
		addSQLiteConfig(prefixes, sqliteFile)
	}
	if kvFile != "" {
		addKVConfig(prefixes, kvFile)
	}
	if s3 != "" {
		if err := addS3Config(prefixesParams, prefixes, s3); err != nil {
			return nil, err
		}
	}
	if googledrive != "" {
		if err := addGoogleDriveConfig(prefixes, googledrive); err != nil {
			return nil, err
		}
	}
	if googlecloudstorage != "" {
		if err := addGoogleCloudStorageConfig(prefixes, googlecloudstorage); err != nil {
			return nil, err
		}
	}
	if swift != "" {
		if err := addSwiftConfig(prefixesParams, prefixes, swift); err != nil {
			return nil, err
		}
	}

	obj["prefixes"] = (map[string]interface{})(prefixes)

	lowLevelConf = &Config{
		Obj:        obj,
		configPath: conf.configPath,
	}
	return lowLevelConf, nil
}
Example #17
0
// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
	var (
		baseURL    = conf.OptionalString("baseURL", "")
		listen     = conf.OptionalString("listen", "")
		auth       = conf.RequiredString("auth")
		keyId      = conf.RequiredString("identity")
		secretRing = conf.RequiredString("identitySecretRing")
		blobPath   = conf.RequiredString("blobPath")
		tlsOn      = conf.OptionalBool("https", false)
		tlsCert    = conf.OptionalString("HTTPSCertFile", "")
		tlsKey     = conf.OptionalString("HTTPSKeyFile", "")
		dbname     = conf.OptionalString("dbname", "")
		mysql      = conf.OptionalString("mysql", "")
		postgres   = conf.OptionalString("postgres", "")
		mongo      = conf.OptionalString("mongo", "")
		_          = conf.OptionalList("replicateTo")
		s3         = conf.OptionalString("s3", "")
		publish    = conf.OptionalObject("publish")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	obj := jsonconfig.Obj{}
	if tlsOn {
		if (tlsCert != "") != (tlsKey != "") {
			return nil, errors.New("Must set both TLSCertFile and TLSKeyFile (or neither to generate a self-signed cert)")
		}
		if tlsCert != "" {
			obj["TLSCertFile"] = tlsCert
			obj["TLSKeyFile"] = tlsKey
		} else {
			obj["TLSCertFile"] = "config/selfgen_cert.pem"
			obj["TLSKeyFile"] = "config/selfgen_key.pem"
		}
	}

	if baseURL != "" {
		if strings.HasSuffix(baseURL, "/") {
			baseURL = baseURL[:len(baseURL)-1]
		}
		obj["baseURL"] = baseURL
	}
	if listen != "" {
		obj["listen"] = listen
	}
	obj["https"] = tlsOn
	obj["auth"] = auth

	if dbname == "" {
		username := os.Getenv("USER")
		if username == "" {
			return nil, fmt.Errorf("USER env var not set; needed to define dbname")
		}
		dbname = "camli" + username
	}

	var indexerPath string
	switch {
	case mongo != "" && mysql != "" || mongo != "" && postgres != "" || mysql != "" && postgres != "":
		return nil, fmt.Errorf("You can only pick one of the db engines (mongo, mysql, postgres).")
	case mysql != "":
		indexerPath = "/index-mysql/"
	case postgres != "":
		indexerPath = "/index-postgres/"
	case mongo != "":
		indexerPath = "/index-mongo/"
	default:
		indexerPath = "/index-mem/"
	}

	entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
	if err != nil {
		return nil, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, err
	}

	prefixesParams := &configPrefixesParams{
		secretRing:  secretRing,
		keyId:       keyId,
		indexerPath: indexerPath,
		blobPath:    blobPath,
		searchOwner: blobref.SHA1FromString(armoredPublicKey),
	}

	prefixes := genLowLevelPrefixes(prefixesParams)
	cacheDir := filepath.Join(blobPath, "/cache")
	if err := os.MkdirAll(cacheDir, 0700); err != nil {
		return nil, fmt.Errorf("Could not create blobs dir %s: %v", cacheDir, err)
	}

	published := []interface{}{}
	if publish != nil {
		published, err = addPublishedConfig(prefixes, publish)
		if err != nil {
			return nil, fmt.Errorf("Could not generate config for published: %v", err)
		}
	}

	addUIConfig(prefixes, "/ui/", published)

	if mysql != "" {
		addMySQLConfig(prefixes, dbname, mysql)
	}
	if postgres != "" {
		addPostgresConfig(prefixes, dbname, postgres)
	}
	if mongo != "" {
		addMongoConfig(prefixes, dbname, mongo)
	}
	if s3 != "" {
		if err := addS3Config(prefixes, s3); err != nil {
			return nil, err
		}
	}
	if indexerPath == "/index-mem/" {
		addMemindexConfig(prefixes)
	}

	obj["prefixes"] = (map[string]interface{})(prefixes)

	lowLevelConf = &Config{
		Obj:        obj,
		configPath: conf.configPath,
	}
	return lowLevelConf, nil
}
Example #18
0
func GenLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
	var (
		baseUrl    = conf.RequiredString("listen")
		auth       = conf.RequiredString("auth")
		keyId      = conf.RequiredString("identity")
		secretRing = conf.RequiredString("identitySecretRing")
		blobPath   = conf.RequiredString("blobPath")
		tlsOn      = conf.OptionalBool("TLS", false)
		dbname     = conf.OptionalString("dbname", "")
		mysql      = conf.OptionalString("mysql", "")
		mongo      = conf.OptionalString("mongo", "")
		_          = conf.OptionalList("replicateTo")
		_          = conf.OptionalString("s3", "")
		publish    = conf.OptionalObject("publish")
	)
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	obj := jsonconfig.Obj{}
	scheme := "http"
	if tlsOn {
		scheme = "https"
		obj["TLSCertFile"] = "config/selfgen_cert.pem"
		obj["TLSKeyFile"] = "config/selfgen_key.pem"
	}
	obj["baseURL"] = scheme + "://" + baseUrl
	obj["https"] = tlsOn
	obj["auth"] = auth

	if dbname == "" {
		username := os.Getenv("USER")
		if username == "" {
			return nil, fmt.Errorf("USER env var not set; needed to define dbname")
		}
		dbname = "camli" + username
	}

	var indexerPath string
	switch {
	case mongo != "" && mysql != "":
		return nil, fmt.Errorf("Cannot have both mysql and mongo in config, pick one")
	case mysql != "":
		indexerPath = "/index-mysql/"
	case mongo != "":
		indexerPath = "/index-mongo/"
	default:
		indexerPath = "/index-mem/"
	}

	entity, err := jsonsign.EntityFromSecring(keyId, secretRing)
	if err != nil {
		return nil, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return nil, err
	}

	prefixesParams := &configPrefixesParams{
		secretRing:  secretRing,
		keyId:       keyId,
		indexerPath: indexerPath,
		blobPath:    blobPath,
		searchOwner: blobref.SHA1FromString(armoredPublicKey),
	}

	prefixes := genLowLevelPrefixes(prefixesParams)
	cacheDir := filepath.Join(blobPath, "/cache")
	if err := os.MkdirAll(cacheDir, 0700); err != nil {
		return nil, fmt.Errorf("Could not create blobs dir %s: %v", cacheDir, err)
	}

	published := []interface{}{}
	if publish != nil {
		published, err = addPublishedConfig(&prefixes, publish)
		if err != nil {
			return nil, fmt.Errorf("Could not generate config for published: %v", err)
		}
	}

	addUIConfig(&prefixes, "/ui/", published)

	if mysql != "" {
		addMysqlConfig(&prefixes, dbname, mysql)
	}
	if mongo != "" {
		addMongoConfig(&prefixes, dbname, mongo)
	}
	if indexerPath == "/index-mem/" {
		addMemindexConfig(&prefixes)
	}

	obj["prefixes"] = (map[string]interface{})(prefixes)

	lowLevelConf = &Config{
		jsonconfig.Obj: obj,
		configPath:     conf.configPath,
	}
	return lowLevelConf, nil
}