func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) {
apiKey := cfg.RequiredString("apiKey")
if err := cfg.Validate(); err != nil {
return nil, err
}
parts := strings.Split(apiKey, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("Foursquare importer: Invalid apiKey configuration: %q", apiKey)
}
clientID, clientSecret := parts[0], parts[1]
im := &imp{
host: host,
tokenCache: &tokenCache{},
imageFileRef: make(map[string]blob.Ref),
oauthConfig: &oauth.Config{
ClientId: clientID,
ClientSecret: clientSecret,
AuthURL: "https://foursquare.com/oauth2/authenticate",
TokenURL: "https://foursquare.com/oauth2/access_token",
RedirectURL: host.BaseURL + "callback",
},
}
// TODO: schedule work?
return im, nil
}
// NewKeyValue returns a KeyValue implementation on top of a
// github.com/cznic/kv file.
func NewKeyValue(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
createOpen := kv.Open
verb := "opening"
if _, err := os.Stat(file); os.IsNotExist(err) {
createOpen = kv.Create
verb = "creating"
}
opts := &kv.Options{
Locker: func(dbname string) (io.Closer, error) {
lkfile := dbname + ".lock"
cl, err := lock.Lock(lkfile)
if err != nil {
return nil, fmt.Errorf("failed to acquire lock on %s: %v", lkfile, err)
}
return cl, nil
},
}
db, err := createOpen(file, opts)
if err != nil {
return nil, fmt.Errorf("error %s %s: %v", verb, file, err)
}
is := &kvis{
db: db,
opts: opts,
path: file,
}
return is, nil
}
func newKeyValueFromConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
return NewKeyValue(file)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix, err := New(kv)
// TODO(mpl): next time we need to do another fix, make a new error
// type that lets us apply the needed fix depending on its value or
// something. For now just one value/fix.
if err == errMissingWholeRef {
// TODO: maybe we don't want to do that automatically. Brad says
// we have to think about the case on GCE/CoreOS in particular.
if err := ix.fixMissingWholeRef(sto); err != nil {
ix.Close()
return nil, fmt.Errorf("could not fix missing wholeRef entries: %v", err)
}
ix, err = New(kv)
}
if err != nil {
return nil, err
}
ix.InitBlobSource(sto)
return ix, err
}
func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
ownerBlobStr := conf.RequiredString("owner")
devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "")
if err := conf.Validate(); err != nil {
return nil, err
}
if devBlockStartupPrefix != "" {
_, err := ld.GetHandler(devBlockStartupPrefix)
if err != nil {
return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err)
}
}
indexHandler, err := ld.GetHandler(indexPrefix)
if err != nil {
return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
}
indexer, ok := indexHandler.(Index)
if !ok {
return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
}
ownerBlobRef, ok := blob.Parse(ownerBlobStr)
if !ok {
return nil, fmt.Errorf("search 'owner' has malformed blobref %q; expecting e.g. sha1-xxxxxxxxxxxx",
ownerBlobStr)
}
return &Handler{
index: indexer,
owner: ownerBlobRef,
}, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &replicaStorage{
replicaPrefixes: config.RequiredList("backends"),
}
nReplicas := len(sto.replicaPrefixes)
sto.minWritesForSuccess = config.OptionalInt("minWritesForSuccess", nReplicas)
if err := config.Validate(); err != nil {
return nil, err
}
if nReplicas == 0 {
return nil, errors.New("replica: need at least one replica")
}
if sto.minWritesForSuccess == 0 {
sto.minWritesForSuccess = nReplicas
}
sto.replicas = make([]blobserver.Storage, nReplicas)
for i, prefix := range sto.replicaPrefixes {
replicaSto, err := ld.GetStorage(prefix)
if err != nil {
return nil, err
}
sto.replicas[i] = replicaSto
}
return sto, nil
}
// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a
// github.com/syndtr/goleveldb/leveldb file.
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
file := cfg.RequiredString("file")
if err := cfg.Validate(); err != nil {
return nil, err
}
strictness := opt.DefaultStrict
if env.IsDev() {
// Be more strict in dev mode.
strictness = opt.StrictAll
}
opts := &opt.Options{
// The default is 10,
// 8 means 2.126% or 1/47th disk check rate,
// 10 means 0.812% error rate (1/2^(bits/1.44)) or 1/123th disk check rate,
// 12 means 0.31% or 1/322th disk check rate.
// TODO(tgulacsi): decide which number is the best here. Till that go with the default.
Filter: filter.NewBloomFilter(10),
Strict: strictness,
}
db, err := leveldb.OpenFile(file, opts)
if err != nil {
return nil, err
}
is := &kvis{
db: db,
path: file,
opts: opts,
readOpts: &opt.ReadOptions{Strict: strictness},
// On machine crash we want to reindex anyway, and
// fsyncs may impose great performance penalty.
writeOpts: &opt.WriteOptions{Sync: false},
}
return is, nil
}
// Reads google storage config and creates a Client. Exits on error.
func doConfig(t *testing.T) (gsa *Client, bucket string) {
if *gsConfigPath == "" {
t.Skip("Skipping manual test. Set flag --gs_config_path to test Google Storage.")
}
cf, err := jsonconfig.ReadFile(*gsConfigPath)
if err != nil {
t.Fatalf("Failed to read config: %v", err)
}
var config jsonconfig.Obj
config = cf.RequiredObject("gsconf")
if err := cf.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
auth := config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
if err := config.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{Scope},
Endpoint: google.Endpoint,
ClientID: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
RedirectURL: oauthutil.TitleBarRedirectURL,
}, auth.RequiredString("refresh_token"))))
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
return
}
func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
u, err := user.Current()
if err != nil {
return
}
root := &RootHandler{
BlobRoot: conf.OptionalString("blobRoot", ""),
SearchRoot: conf.OptionalString("searchRoot", ""),
OwnerName: conf.OptionalString("ownerName", u.Name),
}
root.Stealth = conf.OptionalBool("stealth", false)
if err = conf.Validate(); err != nil {
return
}
if root.BlobRoot != "" {
bs, err := ld.GetStorage(root.BlobRoot)
if err != nil {
return nil, fmt.Errorf("Root handler's blobRoot of %q error: %v", root.BlobRoot, err)
}
root.Storage = bs
}
if root.SearchRoot != "" {
h, _ := ld.GetHandler(root.SearchRoot)
root.Search = h.(*search.Handler)
}
return root, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.Storage, err error) {
sto := &storage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
}
key := config.OptionalString("key", "")
keyFile := config.OptionalString("keyFile", "")
switch {
case key != "":
sto.key = []byte(key)
case keyFile != "":
// TODO: check that keyFile's unix permissions aren't too permissive.
sto.key, err = ioutil.ReadFile(keyFile)
if err != nil {
return
}
}
sto.blobs, err = ld.GetStorage(config.RequiredString("blobs"))
if err != nil {
return
}
sto.meta, err = ld.GetStorage(config.RequiredString("meta"))
if err != nil {
return
}
if err := config.Validate(); err != nil {
return nil, err
}
if sto.key == nil {
// TODO: add a way to prompt from stdin on start? or keychain support?
return nil, errors.New("no encryption key set with 'key' or 'keyFile'")
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
path := config.RequiredString("path")
if err := config.Validate(); err != nil {
return nil, err
}
return New(path)
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
client := &swift.Connection{
UserName: config.RequiredString("user_name"),
Tenant: config.RequiredString("tenant"),
ApiKey: config.RequiredString("secret"),
AuthUrl: config.RequiredString("auth_url"),
}
if err := client.Authenticate(); err != nil {
return nil, fmt.Errorf("Authentication failure on swift: %v", err)
}
sto := &swiftStorage{
client: client,
container: config.OptionalString("container", "default"),
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if !skipStartupCheck {
containers, err := client.ContainerNames(nil)
if err != nil {
return nil, fmt.Errorf("Failed to get container list from swift: %v", err)
}
haveContainer := make(map[string]bool)
for _, c := range containers {
haveContainer[c] = true
}
if !haveContainer[sto.container] {
if err := client.ContainerCreate(sto.container, nil); err != nil {
return nil, fmt.Errorf("Swift bucket %s doesn't exist and it's impossible to create it: ", sto.container, err)
}
}
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
file := config.RequiredString("file")
if err := config.Validate(); err != nil {
return nil, err
}
is, closer, err := NewStorage(file)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
closer.Close()
return nil, err
}
ix := index.New(is)
if err != nil {
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}
// Reads google storage config and creates a Client. Exits on error.
func doConfig(t *testing.T) (gsa *Client, bucket string) {
gsConfigPath := filepath.Join(osutil.CamliConfigDir(), "gstestconfig.json")
if _, err := os.Stat(gsConfigPath); os.IsNotExist(err) {
test.DependencyErrorOrSkip(t)
t.Fatalf("Missing config file: %v", err)
}
cf, err := jsonconfig.ReadFile(gsConfigPath)
if err != nil {
t.Fatalf("Failed to read config: %v", err)
}
var config jsonconfig.Obj
config = cf.RequiredObject("gsconf")
if err := cf.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
auth := config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
if err := config.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(MakeOauthTransport(auth.RequiredString("client_id"),
auth.RequiredString("client_secret"),
auth.RequiredString("refresh_token")))
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
return
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
url := config.RequiredString("url")
auth := config.RequiredString("auth")
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
client := client.New(url)
if err = client.SetupAuthFromString(auth); err != nil {
return nil, err
}
client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags))
sto := &remoteStorage{
client: client,
}
if !skipStartupCheck {
// Do a quick dummy operation to check that our credentials are
// correct.
// TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason.
c := make(chan blob.SizedRef, 1)
err = sto.EnumerateBlobs(context.TODO(), c, "", 1)
if err != nil {
return nil, err
}
}
return sto, nil
}
// Reads google storage config and creates a Client. Exits on error.
func doConfig(t *testing.T) (gsa *Client, bucket string) {
if *gsConfigPath == "" {
t.Skip("Skipping manual test. Set flag --gs_config_path to test Google Storage.")
}
cf, err := jsonconfig.ReadFile(*gsConfigPath)
if err != nil {
t.Fatalf("Failed to read config: %v", err)
}
var config jsonconfig.Obj
config = cf.RequiredObject("gsconf")
if err := cf.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
auth := config.RequiredObject("auth")
bucket = config.RequiredString("bucket")
if err := config.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(MakeOauthTransport(auth.RequiredString("client_id"),
auth.RequiredString("client_secret"),
auth.RequiredString("refresh_token")))
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
}
return
}
func newMongoIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
mgw := &MongoWrapper{
Servers: config.OptionalString("host", "localhost"),
Database: config.RequiredString("database"),
User: config.OptionalString("user", ""),
Password: config.OptionalString("password", ""),
Collection: collectionName,
}
if err := config.Validate(); err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix, err := newMongoIndex(mgw)
if err != nil {
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
if wipe, _ := strconv.ParseBool(os.Getenv("CAMLI_MONGO_WIPE")); wipe {
err = ix.Storage().Delete("")
if err != nil {
return nil, err
}
}
return ix, err
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
auth := config.RequiredObject("auth")
oauthConf := &oauth.Config{
ClientId: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
AuthURL: GoogleOAuth2AuthURL,
TokenURL: GoogleOAuth2TokenURL,
}
// force refreshes the access token on start, make sure
// refresh request in parallel are being started
transport := &oauth.Transport{
Token: &oauth.Token{
AccessToken: "",
RefreshToken: auth.RequiredString("refresh_token"),
Expiry: time.Now(),
},
Config: oauthConf,
Transport: http.DefaultTransport,
}
parent := config.RequiredString("parent_id")
if err := config.Validate(); err != nil {
return nil, err
}
service, err := service.New(transport, parent)
sto := &driveStorage{
service: service,
}
return sto, err
}
func indexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
is := &indexStorage{}
var (
blobPrefix = config.RequiredString("blobSource")
ns = config.OptionalString("namespace", "")
)
if err := config.Validate(); err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
is.ns, err = sanitizeNamespace(ns)
if err != nil {
return nil, err
}
ix, err := index.New(is)
if err != nil {
return nil, err
}
ix.BlobSource = sto
ix.KeyFetcher = ix.BlobSource // TODO(bradfitz): global search? something else?
return ix, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
client := &s3.Client{
Auth: &s3.Auth{
AccessKey: config.RequiredString("aws_access_key"),
SecretAccessKey: config.RequiredString("aws_secret_access_key"),
},
HttpClient: http.DefaultClient,
}
sto := &s3Storage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
s3Client: client,
bucket: config.RequiredString("bucket"),
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if !skipStartupCheck {
// TODO: skip this check if a file
// ~/.camli/.configcheck/sha1-("IS GOOD: s3: sha1(access key +
// secret key)") exists and has recent time?
buckets, err := client.Buckets()
if err != nil {
return nil, fmt.Errorf("Failed to get bucket list from S3: %v", err)
}
haveBucket := make(map[string]bool)
for _, b := range buckets {
haveBucket[b.Name] = true
}
if !haveBucket[sto.bucket] {
return nil, fmt.Errorf("S3 bucket %q doesn't exist. Create it first at https://console.aws.amazon.com/s3/home")
}
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
ix, err := New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &condStorage{}
receive := conf.OptionalStringOrObject("write")
read := conf.RequiredString("read")
remove := conf.OptionalString("remove", "")
if err := conf.Validate(); err != nil {
return nil, err
}
if receive != nil {
sto.storageForReceive, err = buildStorageForReceive(ld, receive)
if err != nil {
return
}
}
sto.read, err = ld.GetStorage(read)
if err != nil {
return
}
if remove != "" {
sto.remove, err = ld.GetStorage(remove)
if err != nil {
return
}
}
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
client := &s3.Client{
Auth: &s3.Auth{
AccessKey: config.RequiredString("aws_access_key"),
SecretAccessKey: config.RequiredString("aws_secret_access_key"),
},
HttpClient: http.DefaultClient,
}
sto := &s3Storage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
s3Client: client,
bucket: config.RequiredString("bucket"),
}
skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
if err := config.Validate(); err != nil {
return nil, err
}
if !skipStartupCheck {
// TODO: skip this check if a file
// ~/.camli/.configcheck/sha1-("IS GOOD: s3: sha1(access key +
// secret key)") exists and has recent time?
if _, err := client.Buckets(); err != nil {
return nil, fmt.Errorf("Failed to get bucket list from S3: %v", err)
}
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
origin = config.RequiredString("origin")
cache = config.RequiredString("cache")
kvConf = config.RequiredObject("meta")
maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20)
)
if err := config.Validate(); err != nil {
return nil, err
}
cacheSto, err := ld.GetStorage(cache)
if err != nil {
return nil, err
}
originSto, err := ld.GetStorage(origin)
if err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConf)
if err != nil {
return nil, err
}
// TODO: enumerate through kv and calculate current size.
// Maybe also even enumerate through cache to see if they match.
// Or even: keep it only in memory and not in kv?
s := &sto{
origin: originSto,
cache: cacheSto,
maxCacheBytes: maxCacheBytes,
kv: kv,
}
return s, nil
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
conninfo := fmt.Sprintf("user=%s dbname=%s host=%s password=%s sslmode=%s",
cfg.RequiredString("user"),
cfg.RequiredString("database"),
cfg.OptionalString("host", "localhost"),
cfg.OptionalString("password", ""),
cfg.OptionalString("sslmode", "require"),
)
if err := cfg.Validate(); err != nil {
return nil, err
}
db, err := sql.Open("postgres", conninfo)
if err != nil {
return nil, err
}
for _, tableSql := range SQLCreateTables() {
if _, err := db.Exec(tableSql); err != nil {
return nil, fmt.Errorf("error creating table with %q: %v", tableSql, err)
}
}
for _, statement := range SQLDefineReplace() {
if _, err := db.Exec(statement); err != nil {
return nil, fmt.Errorf("error setting up replace statement with %q: %v", statement, err)
}
}
r, err := db.Query(fmt.Sprintf(`SELECT replaceintometa('version', '%d')`, SchemaVersion()))
if err != nil {
return nil, fmt.Errorf("error setting schema version: %v", err)
}
r.Close()
kv := &keyValue{
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
SetFunc: altSet,
BatchSetFunc: altBatchSet,
PlaceHolderFunc: replacePlaceHolders,
},
}
if err := kv.ping(); err != nil {
return nil, fmt.Errorf("PostgreSQL db unreachable: %v", err)
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
}
if version != requiredSchemaVersion {
if env.IsDev() {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) {
host := cfg.OptionalString("host", "")
dsn := fmt.Sprintf("%s/%s/%s",
cfg.RequiredString("database"),
cfg.RequiredString("user"),
cfg.OptionalString("password", ""),
)
if err := cfg.Validate(); err != nil {
return nil, err
}
if host != "" {
// TODO(mpl): document that somewhere
if !strings.Contains(host, ":") {
host = host + ":3306"
}
dsn = "tcp:" + host + "*" + dsn
}
db, err := sql.Open("mymysql", dsn)
if err != nil {
return nil, err
}
for _, tableSql := range SQLCreateTables() {
if _, err := db.Exec(tableSql); err != nil {
return nil, fmt.Errorf("error creating table with %q: %v", tableSql, err)
}
}
if _, err := db.Exec(fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, SchemaVersion())); err != nil {
return nil, fmt.Errorf("error setting schema version: %v", err)
}
kv := &keyValue{
db: db,
KeyValue: &sqlkv.KeyValue{
DB: db,
},
}
if err := kv.ping(); err != nil {
return nil, fmt.Errorf("MySQL db unreachable: %v", err)
}
version, err := kv.SchemaVersion()
if err != nil {
return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
}
if version != requiredSchemaVersion {
if version == 20 && requiredSchemaVersion == 21 {
fmt.Fprintf(os.Stderr, fixSchema20to21)
}
if os.Getenv("CAMLI_DEV_CAMLI_ROOT") != "" {
// Good signal that we're using the devcam server, so help out
// the user with a more useful tip:
return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
}
return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
version, requiredSchemaVersion)
}
return kv, nil
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
from := conf.RequiredString("from")
to := conf.RequiredString("to")
fullSync := conf.OptionalBool("fullSyncOnStart", false)
blockFullSync := conf.OptionalBool("blockingFullSyncOnStart", false)
if err = conf.Validate(); err != nil {
return
}
fromBs, err := ld.GetStorage(from)
if err != nil {
return
}
toBs, err := ld.GetStorage(to)
if err != nil {
return
}
fromQsc, ok := fromBs.(blobserver.StorageQueueCreator)
if !ok {
return nil, fmt.Errorf("Prefix %s (type %T) does not support being efficient replication source (queueing)", from, fromBs)
}
synch, err := createSyncHandler(from, to, fromQsc, toBs)
if err != nil {
return
}
if fullSync || blockFullSync {
didFullSync := make(chan bool, 1)
go func() {
n := synch.runSync("queue", fromQsc, 0)
log.Printf("Queue sync copied %d blobs", n)
n = synch.runSync("full", fromBs, 0)
log.Printf("Full sync copied %d blobs", n)
didFullSync <- true
synch.syncQueueLoop()
}()
if blockFullSync {
log.Printf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
log.Printf("Full sync complete.")
}
} else {
go synch.syncQueueLoop()
}
rootPrefix, _, err := ld.FindHandlerByType("root")
switch err {
case blobserver.ErrHandlerTypeNotFound:
// ignore; okay to not have a root handler.
case nil:
h, err := ld.GetHandler(rootPrefix)
if err != nil {
return nil, err
}
h.(*RootHandler).registerSyncHandler(synch)
default:
return nil, fmt.Errorf("Error looking for root handler: %v", err)
}
return synch, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.Storage, err error) {
sto := &storage{
SimpleBlobHubPartitionMap: &blobserver.SimpleBlobHubPartitionMap{},
index: index.NewMemoryStorage(), // TODO: temporary for development; let be configurable (mysql, etc)
}
agreement := config.OptionalString("I_AGREE", "")
const wantAgreement = "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change."
if agreement != wantAgreement {
return nil, errors.New("Use of the 'encrypt' target without the proper I_AGREE value.")
}
key := config.OptionalString("key", "")
keyFile := config.OptionalString("keyFile", "")
var keyb []byte
switch {
case key != "":
keyb, err = hex.DecodeString(key)
if err != nil || len(keyb) != 16 {
return nil, fmt.Errorf("The 'key' parameter must be 16 bytes of 32 hex digits. (currently fixed at AES-128)")
}
case keyFile != "":
// TODO: check that keyFile's unix permissions aren't too permissive.
keyb, err = ioutil.ReadFile(keyFile)
if err != nil {
return nil, fmt.Errorf("Reading key file %v: %v", keyFile, err)
}
}
blobStorage := config.RequiredString("blobs")
metaStorage := config.RequiredString("meta")
if err := config.Validate(); err != nil {
return nil, err
}
sto.blobs, err = ld.GetStorage(blobStorage)
if err != nil {
return
}
sto.meta, err = ld.GetStorage(metaStorage)
if err != nil {
return
}
if keyb == nil {
// TODO: add a way to prompt from stdin on start? or keychain support?
return nil, errors.New("no encryption key set with 'key' or 'keyFile'")
}
if err := sto.setKey(keyb); err != nil {
return nil, err
}
log.Printf("Reading encryption metadata...")
if err := sto.readAllMetaBlobs(); err != nil {
return nil, fmt.Errorf("Error scanning metadata on start-up: %v", err)
}
log.Printf("Read all encryption metadata.")
return sto, nil
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
path := config.RequiredString("path")
maxFileSize := config.OptionalInt("maxFileSize", 0)
if err := config.Validate(); err != nil {
return nil, err
}
return newStorage(path, int64(maxFileSize))
}
func newStatusFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
if err := conf.Validate(); err != nil {
return nil, err
}
return &StatusHandler{
prefix: ld.MyPrefix(),
handlerFinder: ld,
}, nil
}