func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { var ( origin = config.RequiredString("origin") cache = config.RequiredString("cache") kvConf = config.RequiredObject("meta") maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20) ) if err := config.Validate(); err != nil { return nil, err } cacheSto, err := ld.GetStorage(cache) if err != nil { return nil, err } originSto, err := ld.GetStorage(origin) if err != nil { return nil, err } kv, err := sorted.NewKeyValue(kvConf) if err != nil { return nil, err } // TODO: enumerate through kv and calculate current size. // Maybe also even enumerate through cache to see if they match. // Or even: keep it only in memory and not in kv? s := &sto{ origin: originSto, cache: cacheSto, maxCacheBytes: maxCacheBytes, kv: kv, } return s, nil }
// convertToMultiServers takes an old style single-server client configuration and maps it to new a multi-servers configuration that is returned. func convertToMultiServers(conf jsonconfig.Obj) (jsonconfig.Obj, error) { server := conf.OptionalString("server", "") if server == "" { return nil, errors.New("Could not convert config to multi-servers style: no \"server\" key found.") } newConf := jsonconfig.Obj{ "servers": map[string]interface{}{ "server": map[string]interface{}{ "auth": conf.OptionalString("auth", ""), "default": true, "server": server, }, }, "identity": conf.OptionalString("identity", ""), "identitySecretRing": conf.OptionalString("identitySecretRing", ""), } if ignoredFiles := conf.OptionalList("ignoredFiles"); ignoredFiles != nil { var list []interface{} for _, v := range ignoredFiles { list = append(list, v) } newConf["ignoredFiles"] = list } return newConf, nil }
// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a // github.com/syndtr/goleveldb/leveldb file. func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) { file := cfg.RequiredString("file") if err := cfg.Validate(); err != nil { return nil, err } strictness := opt.DefaultStrict if env.IsDev() { // Be more strict in dev mode. strictness = opt.StrictAll } opts := &opt.Options{ // The default is 10, // 8 means 2.126% or 1/47th disk check rate, // 10 means 0.812% error rate (1/2^(bits/1.44)) or 1/123th disk check rate, // 12 means 0.31% or 1/322th disk check rate. // TODO(tgulacsi): decide which number is the best here. Till that go with the default. Filter: filter.NewBloomFilter(10), Strict: strictness, } db, err := leveldb.OpenFile(file, opts) if err != nil { return nil, err } is := &kvis{ db: db, path: file, opts: opts, readOpts: &opt.ReadOptions{Strict: strictness}, // On machine crash we want to reindex anyway, and // fsyncs may impose great performance penalty. writeOpts: &opt.WriteOptions{Sync: false}, } return is, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { path := config.RequiredString("path") if err := config.Validate(); err != nil { return nil, err } return New(path) }
// printConfigChangeHelp checks if conf contains obsolete keys, // and prints additional help in this case. func printConfigChangeHelp(conf jsonconfig.Obj) { // rename maps from old key names to the new ones. // If there is no new one, the value is the empty string. rename := map[string]string{ "keyId": "identity", "publicKeyBlobref": "", "selfPubKeyDir": "", "secretRing": "identitySecretRing", } oldConfig := false configChangedMsg := fmt.Sprintf("The client configuration file (%s) keys have changed.\n", osutil.UserClientConfigPath()) for _, unknown := range conf.UnknownKeys() { v, ok := rename[unknown] if ok { if v != "" { configChangedMsg += fmt.Sprintf("%q should be renamed %q.\n", unknown, v) } else { configChangedMsg += fmt.Sprintf("%q should be removed.\n", unknown) } oldConfig = true } } if oldConfig { configChangedMsg += "Please see http://camlistore.org/docs/client-config, or use camput init to recreate a default one." log.Print(configChangedMsg) } }
func thumbnailerFromConfig(config jsonconfig.Obj) Thumbnailer { command := config.OptionalList("command") if len(command) < 1 { return DefaultThumbnailer } return &configThumbnailer{prog: command[0], args: command[1:]} }
// detectConfigChange returns an informative error if conf contains obsolete keys. func detectConfigChange(conf jsonconfig.Obj) error { oldHTTPSKey, oldHTTPSCert := conf.OptionalString("HTTPSKeyFile", ""), conf.OptionalString("HTTPSCertFile", "") if oldHTTPSKey != "" || oldHTTPSCert != "" { return fmt.Errorf("Config keys %q and %q have respectively been renamed to %q and %q, please fix your server config.", "HTTPSKeyFile", "HTTPSCertFile", "httpsKey", "httpsCert") } return nil }
func newStatusFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) { if err := conf.Validate(); err != nil { return nil, err } return &StatusHandler{ prefix: ld.MyPrefix(), handlerFinder: ld, }, nil }
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { sto := &appengineStorage{ namespace: config.OptionalString("namespace", ""), } if err := config.Validate(); err != nil { return nil, err } sto.namespace, err = sanitizeNamespace(sto.namespace) if err != nil { return nil, err } return sto, nil }
func newKeyValueFromConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) { if !compiled { return nil, ErrNotCompiled } file := cfg.RequiredString("file") if err := cfg.Validate(); err != nil { return nil, err } fi, err := os.Stat(file) if os.IsNotExist(err) || (err == nil && fi.Size() == 0) { if err := initDB(file); err != nil { return nil, fmt.Errorf("could not initialize sqlite DB at %s: %v", file, err) } } db, err := sql.Open("sqlite3", file) if err != nil { return nil, err } kv := &keyValue{ file: file, db: db, KeyValue: &sqlkv.KeyValue{ DB: db, Gate: syncutil.NewGate(1), }, } version, err := kv.SchemaVersion() if err != nil { return nil, fmt.Errorf("error getting schema version (need to init database with 'camtool dbinit %s'?): %v", file, err) } if err := kv.ping(); err != nil { return nil, err } if version != requiredSchemaVersion { if env.IsDev() { // Good signal that we're using the devcam server, so help out // the user with a more useful tip: return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) } return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)", version, requiredSchemaVersion) } return kv, nil }
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (storage blobserver.Storage, err error) { sto := &condStorage{} receive := conf.OptionalStringOrObject("write") read := conf.RequiredString("read") remove := conf.OptionalString("remove", "") if err := conf.Validate(); err != nil { return nil, err } if receive != nil { sto.storageForReceive, err = buildStorageForReceive(ld, receive) if err != nil { return } } sto.read, err = ld.GetStorage(read) if err != nil { return } if remove != "" { sto.remove, err = ld.GetStorage(remove) if err != nil { return } } return sto, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { url := config.RequiredString("url") auth := config.RequiredString("auth") skipStartupCheck := config.OptionalBool("skipStartupCheck", false) if err := config.Validate(); err != nil { return nil, err } client := client.New(url) if err = client.SetupAuthFromString(auth); err != nil { return nil, err } client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags)) sto := &remoteStorage{ client: client, } if !skipStartupCheck { // Do a quick dummy operation to check that our credentials are // correct. // TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason. c := make(chan blob.SizedRef, 1) err = sto.EnumerateBlobs(context.TODO(), c, "", 1) if err != nil { return nil, err } } return sto, nil }
func newMemoryIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { blobPrefix := config.RequiredString("blobSource") if err := config.Validate(); err != nil { return nil, err } sto, err := ld.GetStorage(blobPrefix) if err != nil { return nil, err } ix := NewMemoryIndex() ix.InitBlobSource(sto) return ix, err }
func NewKeyValue(cfg jsonconfig.Obj) (KeyValue, error) { var s KeyValue var err error typ := cfg.RequiredString("type") ctor, ok := ctors[typ] if typ != "" && !ok { return nil, fmt.Errorf("Invalid sorted.KeyValue type %q", typ) } if ok { s, err = ctor(cfg) if err != nil { return nil, fmt.Errorf("error from %q KeyValue: %v", typ, err) } } return s, cfg.Validate() }
func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) { ins := &instance{ server: cfg.OptionalString("host", "localhost"), database: cfg.RequiredString("database"), user: cfg.OptionalString("user", ""), password: cfg.OptionalString("password", ""), } if err := cfg.Validate(); err != nil { return nil, err } db, err := ins.getCollection() if err != nil { return nil, err } return &keyValue{db: db, session: ins.session}, nil }
// newKeyValueFromJSONConfig returns a KeyValue implementation on top of a // github.com/cznic/kv file. func newKeyValueFromJSONConfig(cfg jsonconfig.Obj) (sorted.KeyValue, error) { file := cfg.RequiredString("file") if err := cfg.Validate(); err != nil { return nil, err } opts := &kv.Options{} db, err := kvutil.Open(file, opts) if err != nil { return nil, err } is := &kvis{ db: db, opts: opts, path: file, } return is, nil }
// NewHandler returns a Handler that proxies requests to an app. Start() on the // Handler starts the app. // The apiHost must end in a slash and is the camlistored API server for the app // process to hit. // The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted. // It must end in a slash, and be at minimum "/". // The conf object has the following members, related to the vars described in // doc/app-environment.txt: // "program", string, required. File name of the app's program executable. Either // an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH. // "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL. // "appConfig", object, optional. Additional configuration that the app can request from Camlistore. func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) { // TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available. name := conf.RequiredString("program") backendURL := conf.OptionalString("backendURL", "") appConfig := conf.OptionalObject("appConfig") // TODO(mpl): add an auth token in the extra config of the dev server config, // that the hello app can use to setup a status handler than only responds // to requests with that token. if err := conf.Validate(); err != nil { return nil, err } if apiHost == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name) } if appHandlerPrefix == "" { return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name) } if backendURL == "" { var err error // If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL // now (instead of letting the app itself do it), because we need to know it in advance in order // to set the app handler's proxy. backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix) if err != nil { return nil, err } } username, password := auth.RandToken(20), auth.RandToken(20) camliAuth := username + ":" + password basicAuth := auth.NewBasicAuth(username, password) envVars := map[string]string{ "CAMLI_API_HOST": apiHost, "CAMLI_AUTH": camliAuth, "CAMLI_APP_BACKEND_URL": backendURL, } if appConfig != nil { envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json" } proxyURL, err := url.Parse(backendURL) if err != nil { return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err) } return &Handler{ name: name, envVars: envVars, auth: basicAuth, appConfig: appConfig, proxy: httputil.NewSingleHostReverseProxy(proxyURL), backendURL: backendURL, }, nil }
// Reads google storage config and creates a Client. Exits on error. func doConfig(t *testing.T) (gsa *Client, bucket string) { if *gsConfigPath == "" { t.Skip("Skipping manual test. Set flag --gs_config_path to test Google Storage.") } cf, err := osutil.NewJSONConfigParser().ReadFile(*gsConfigPath) if err != nil { t.Fatalf("Failed to read config: %v", err) } var config jsonconfig.Obj config = cf.RequiredObject("gsconf") if err := cf.Validate(); err != nil { t.Fatalf("Invalid config: %v", err) } auth := config.RequiredObject("auth") bucket = config.RequiredString("bucket") if err := config.Validate(); err != nil { t.Fatalf("Invalid config: %v", err) } gsa = NewClient(oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{ Scopes: []string{Scope}, Endpoint: google.Endpoint, ClientID: auth.RequiredString("client_id"), ClientSecret: auth.RequiredString("client_secret"), RedirectURL: oauthutil.TitleBarRedirectURL, }, auth.RequiredString("refresh_token")))) if err := auth.Validate(); err != nil { t.Fatalf("Invalid config: %v", err) } return }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { var ( path = config.RequiredString("path") maxFileSize = config.OptionalInt("maxFileSize", 0) indexConf = config.OptionalObject("metaIndex") ) if err := config.Validate(); err != nil { return nil, err } return newStorage(path, int64(maxFileSize), indexConf) }
func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here? ownerBlobStr := conf.RequiredString("owner") devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "") slurpToMemory := conf.OptionalBool("slurpToMemory", false) if err := conf.Validate(); err != nil { return nil, err } if devBlockStartupPrefix != "" { _, err := ld.GetHandler(devBlockStartupPrefix) if err != nil { return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err) } } indexHandler, err := ld.GetHandler(indexPrefix) if err != nil { return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix) } indexer, ok := indexHandler.(index.Interface) if !ok { return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler) } ownerBlobRef, ok := blob.Parse(ownerBlobStr) if !ok { return nil, fmt.Errorf("search 'owner' has malformed blobref %q; expecting e.g. sha1-xxxxxxxxxxxx", ownerBlobStr) } h := NewHandler(indexer, ownerBlobRef) if slurpToMemory { ii := indexer.(*index.Index) ii.Lock() corpus, err := ii.KeepInMemory() if err != nil { ii.Unlock() return nil, fmt.Errorf("error slurping index to memory: %v", err) } h.SetCorpus(corpus) ii.Unlock() } return h, nil }
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { sto := &shardStorage{ shardPrefixes: config.RequiredList("backends"), } if err := config.Validate(); err != nil { return nil, err } if len(sto.shardPrefixes) == 0 { return nil, errors.New("shard: need at least one shard") } sto.shards = make([]blobserver.Storage, len(sto.shardPrefixes)) for i, prefix := range sto.shardPrefixes { shardSto, err := ld.GetStorage(prefix) if err != nil { return nil, err } sto.shards[i] = shardSto } return sto, nil }
func indexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { is := &indexStorage{} var ( blobPrefix = config.RequiredString("blobSource") ns = config.OptionalString("namespace", "") ) if err := config.Validate(); err != nil { return nil, err } sto, err := ld.GetStorage(blobPrefix) if err != nil { return nil, err } is.ns, err = sanitizeNamespace(ns) if err != nil { return nil, err } ix, err := index.New(is) if err != nil { return nil, err } ix.BlobSource = sto ix.KeyFetcher = ix.BlobSource // TODO(bradfitz): global search? something else? return ix, nil }
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { blobPrefix := config.RequiredString("blobSource") kvConfig := config.RequiredObject("storage") if err := config.Validate(); err != nil { return nil, err } kv, err := sorted.NewKeyValue(kvConfig) if err != nil { return nil, err } sto, err := ld.GetStorage(blobPrefix) if err != nil { return nil, err } ix, err := New(kv) // TODO(mpl): next time we need to do another fix, make a new error // type that lets us apply the needed fix depending on its value or // something. For now just one value/fix. if err == errMissingWholeRef { // TODO: maybe we don't want to do that automatically. Brad says // we have to think about the case on GCE/CoreOS in particular. if err := ix.fixMissingWholeRef(sto); err != nil { ix.Close() return nil, fmt.Errorf("could not fix missing wholeRef entries: %v", err) } ix, err = New(kv) } if err != nil { return nil, err } ix.InitBlobSource(sto) return ix, err }
func newJSONSignFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { var ( // either a short form ("26F5ABDA") or one the longer forms. keyId = conf.RequiredString("keyId") pubKeyDestPrefix = conf.OptionalString("publicKeyDest", "") secretRing = conf.OptionalString("secretRing", "") ) if err := conf.Validate(); err != nil { return nil, err } h := &Handler{ secretRing: secretRing, } var err error h.entity, err = jsonsign.EntityFromSecring(keyId, h.secretRingPath()) if err != nil { return nil, err } h.pubKey, err = jsonsign.ArmoredPublicKey(h.entity) ms := &memory.Storage{} h.pubKeyBlobRef = blob.SHA1FromString(h.pubKey) if _, err := ms.ReceiveBlob(h.pubKeyBlobRef, strings.NewReader(h.pubKey)); err != nil { return nil, fmt.Errorf("could not store pub key blob: %v", err) } h.pubKeyFetcher = ms if pubKeyDestPrefix != "" { sto, err := ld.GetStorage(pubKeyDestPrefix) if err != nil { return nil, err } h.pubKeyDest = sto } h.pubKeyBlobRefServeSuffix = "camli/" + h.pubKeyBlobRef.String() h.pubKeyHandler = &gethandler.Handler{ Fetcher: ms, } h.signer, err = schema.NewSigner(h.pubKeyBlobRef, strings.NewReader(h.pubKey), h.entity) if err != nil { return nil, err } return h, nil }
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) { hc := HostConfig{ BaseURL: ld.BaseURL(), Prefix: ld.MyPrefix(), } ClientId := make(map[string]string) ClientSecret := make(map[string]string) for k, _ := range importers { var clientId, clientSecret string if impConf := cfg.OptionalObject(k); impConf != nil { clientId = impConf.OptionalString("clientID", "") clientSecret = impConf.OptionalString("clientSecret", "") // Special case: allow clientSecret to be of form "clientId:clientSecret" // if the clientId is empty. if clientId == "" && strings.Contains(clientSecret, ":") { if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 { clientId, clientSecret = f[0], f[1] } } if err := impConf.Validate(); err != nil { return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err) } ClientId[k] = clientId ClientSecret[k] = clientSecret } } if err := cfg.Validate(); err != nil { return nil, err } hc.ClientId = ClientId hc.ClientSecret = ClientSecret host, err := NewHost(hc) if err != nil { return nil, err } host.didInit.Add(1) return host, nil }
func newShareFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) { blobRoot := conf.RequiredString("blobRoot") if blobRoot == "" { return nil, errors.New("No blobRoot defined for share handler") } if err := conf.Validate(); err != nil { return nil, err } bs, err := ld.GetStorage(blobRoot) if err != nil { return nil, fmt.Errorf("failed to get share handler's storage at %q: %v", blobRoot, err) } fetcher, ok := bs.(blob.Fetcher) if !ok { return nil, errors.New("share handler's storage not a Fetcher.") } sh := &shareHandler{ fetcher: fetcher, log: true, } return sh, nil }
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { var ( auth = config.RequiredObject("auth") bucket = config.RequiredString("bucket") cacheSize = config.OptionalInt64("cacheSize", 32<<20) accountID = auth.RequiredString("account_id") appKey = auth.RequiredString("application_key") ) if err := config.Validate(); err != nil { return nil, err } if err := auth.Validate(); err != nil { return nil, err } var dirPrefix string if parts := strings.SplitN(bucket, "/", 2); len(parts) > 1 { dirPrefix = parts[1] bucket = parts[0] } if dirPrefix != "" && !strings.HasSuffix(dirPrefix, "/") { dirPrefix += "/" } cl, err := b2.NewClient(accountID, appKey, nil) if err != nil { return nil, err } b, err := cl.BucketByName(bucket, true) if err != nil { return nil, err } s := &Storage{ cl: cl, b: b, dirPrefix: dirPrefix, } if cacheSize != 0 { s.cache = memory.NewCache(cacheSize) } return s, nil }
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) { var ( smallPrefix = conf.RequiredString("smallBlobs") largePrefix = conf.RequiredString("largeBlobs") metaConf = conf.RequiredObject("metaIndex") ) if err := conf.Validate(); err != nil { return nil, err } small, err := ld.GetStorage(smallPrefix) if err != nil { return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err) } large, err := ld.GetStorage(largePrefix) if err != nil { return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err) } largeSubber, ok := large.(subFetcherStorage) if !ok { return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs", largePrefix, large) } meta, err := sorted.NewKeyValue(metaConf) if err != nil { return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err) } sto := &storage{ small: small, large: largeSubber, meta: meta, } sto.init() recoveryMu.Lock() defer recoveryMu.Unlock() if recovery { log.Print("Starting recovery of blobpacked index") if err := meta.Close(); err != nil { return nil, err } if err := sto.reindex(context.TODO(), func() (sorted.KeyValue, error) { return sorted.NewKeyValue(metaConf) }); err != nil { return nil, err } return sto, nil } // Check for a weird state: zip files exist, but no metadata about them // is recorded. This is probably a corrupt state, and the user likely // wants to recover. if !sto.anyMeta() && sto.anyZipPacks() { log.Fatal("Error: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode with -recovery.") } return sto, nil }
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) { var ( smallPrefix = conf.RequiredString("smallBlobs") largePrefix = conf.RequiredString("largeBlobs") metaConf = conf.RequiredObject("metaIndex") ) if err := conf.Validate(); err != nil { return nil, err } small, err := ld.GetStorage(smallPrefix) if err != nil { return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err) } large, err := ld.GetStorage(largePrefix) if err != nil { return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err) } largeSubber, ok := large.(subFetcherStorage) if !ok { return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs", largePrefix, large) } meta, err := sorted.NewKeyValue(metaConf) if err != nil { return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err) } sto := &storage{ small: small, large: largeSubber, meta: meta, } sto.init() // Check for a weird state: zip files exist, but no metadata about them // is recorded. This is probably a corrupt state, and the user likely // wants to recover. if !sto.anyMeta() && sto.anyZipPacks() { log.Printf("Warning: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode.") // TODO: add a recovery mode. // Old TODO was: // fail with a "known corrupt" message and refuse to // start unless in recovery mode (perhaps a new environment // var? or flag passed down?) using StreamBlobs starting at // "l:". Could even do it automatically if total size is // small or fast enough? But that's confusing if it only // sometimes finishes recovery. We probably want various // server start-up modes anyway: "check", "recover", "garbage // collect", "readonly". So might as well introduce that // concept now. // TODO: test start-up recovery mode, once it works. } return sto, nil }
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { sto := &replicaStorage{ replicaPrefixes: config.RequiredList("backends"), readPrefixes: config.OptionalList("readBackends"), } nReplicas := len(sto.replicaPrefixes) sto.minWritesForSuccess = config.OptionalInt("minWritesForSuccess", nReplicas) if err := config.Validate(); err != nil { return nil, err } if nReplicas == 0 { return nil, errors.New("replica: need at least one replica") } if sto.minWritesForSuccess == 0 { sto.minWritesForSuccess = nReplicas } // readPrefixes defaults to the write prefixes. if len(sto.readPrefixes) == 0 { sto.readPrefixes = sto.replicaPrefixes } for _, prefix := range sto.replicaPrefixes { s, err := ld.GetStorage(prefix) if err != nil { // If it's not a storage interface, it might be an http Handler // that also supports being a target (e.g. a sync handler). h, _ := ld.GetHandler(prefix) var ok bool if s, ok = h.(blobserver.Storage); !ok { return nil, err } } sto.replicas = append(sto.replicas, s) } for _, prefix := range sto.readPrefixes { s, err := ld.GetStorage(prefix) if err != nil { return nil, err } sto.readReplicas = append(sto.readReplicas, s) } return sto, nil }