// newIndex returns a new sorted.KeyValue, using either the given config, or the default.
func newIndex(root string, indexConf jsonconfig.Obj) (sorted.KeyValue, error) {
if len(indexConf) > 0 {
return sorted.NewKeyValue(indexConf)
}
return sorted.NewKeyValue(jsonconfig.Obj{
"type": defaultIndexType,
"file": filepath.Join(root, defaultIndexFile),
})
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) {
var (
smallPrefix = conf.RequiredString("smallBlobs")
largePrefix = conf.RequiredString("largeBlobs")
metaConf = conf.RequiredObject("metaIndex")
)
if err := conf.Validate(); err != nil {
return nil, err
}
small, err := ld.GetStorage(smallPrefix)
if err != nil {
return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err)
}
large, err := ld.GetStorage(largePrefix)
if err != nil {
return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err)
}
largeSubber, ok := large.(subFetcherStorage)
if !ok {
return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs",
largePrefix, large)
}
meta, err := sorted.NewKeyValue(metaConf)
if err != nil {
return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err)
}
sto := &storage{
small: small,
large: largeSubber,
meta: meta,
}
sto.init()
recoveryMu.Lock()
defer recoveryMu.Unlock()
if recovery {
log.Print("Starting recovery of blobpacked index")
if err := meta.Close(); err != nil {
return nil, err
}
if err := sto.reindex(context.TODO(), func() (sorted.KeyValue, error) {
return sorted.NewKeyValue(metaConf)
}); err != nil {
return nil, err
}
return sto, nil
}
// Check for a weird state: zip files exist, but no metadata about them
// is recorded. This is probably a corrupt state, and the user likely
// wants to recover.
if !sto.anyMeta() && sto.anyZipPacks() {
log.Fatal("Error: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode with -recovery.")
}
return sto, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
ix, err := New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix, err := New(kv)
// TODO(mpl): next time we need to do another fix, make a new error
// type that lets us apply the needed fix depending on its value or
// something. For now just one value/fix.
if err == errMissingWholeRef {
// TODO: maybe we don't want to do that automatically. Brad says
// we have to think about the case on GCE/CoreOS in particular.
if err := ix.fixMissingWholeRef(sto); err != nil {
ix.Close()
return nil, fmt.Errorf("could not fix missing wholeRef entries: %v", err)
}
ix, err = New(kv)
}
if err != nil {
return nil, err
}
ix.InitBlobSource(sto)
return ix, err
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
var (
origin = config.RequiredString("origin")
cache = config.RequiredString("cache")
kvConf = config.RequiredObject("meta")
maxCacheBytes = config.OptionalInt64("maxCacheBytes", 512<<20)
)
if err := config.Validate(); err != nil {
return nil, err
}
cacheSto, err := ld.GetStorage(cache)
if err != nil {
return nil, err
}
originSto, err := ld.GetStorage(origin)
if err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConf)
if err != nil {
return nil, err
}
// TODO: enumerate through kv and calculate current size.
// Maybe also even enumerate through cache to see if they match.
// Or even: keep it only in memory and not in kv?
s := &sto{
origin: originSto,
cache: cacheSto,
maxCacheBytes: maxCacheBytes,
kv: kv,
}
return s, nil
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
f, err := ioutil.TempFile("", "sqlite-test")
if err != nil {
t.Fatal(err)
}
db, err := sql.Open("sqlite3", f.Name())
if err != nil {
t.Fatalf("opening test database: %v", err)
}
for _, tableSql := range sqlite.SQLCreateTables() {
do(db, tableSql)
}
do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, sqlite.SchemaVersion()))
kv, err = sorted.NewKeyValue(jsonconfig.Obj{
"type": "sqlite",
"file": f.Name(),
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
os.Remove(f.Name())
}
}
func newFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (blobserver.Storage, error) {
var (
smallPrefix = conf.RequiredString("smallBlobs")
largePrefix = conf.RequiredString("largeBlobs")
metaConf = conf.RequiredObject("metaIndex")
)
if err := conf.Validate(); err != nil {
return nil, err
}
small, err := ld.GetStorage(smallPrefix)
if err != nil {
return nil, fmt.Errorf("failed to load smallBlobs at %s: %v", smallPrefix, err)
}
large, err := ld.GetStorage(largePrefix)
if err != nil {
return nil, fmt.Errorf("failed to load largeBlobs at %s: %v", largePrefix, err)
}
largeSubber, ok := large.(subFetcherStorage)
if !ok {
return nil, fmt.Errorf("largeBlobs at %q of type %T doesn't support fetching sub-ranges of blobs",
largePrefix, large)
}
meta, err := sorted.NewKeyValue(metaConf)
if err != nil {
return nil, fmt.Errorf("failed to setup blobpacked metaIndex: %v", err)
}
sto := &storage{
small: small,
large: largeSubber,
meta: meta,
}
sto.init()
// Check for a weird state: zip files exist, but no metadata about them
// is recorded. This is probably a corrupt state, and the user likely
// wants to recover.
if !sto.anyMeta() && sto.anyZipPacks() {
log.Printf("Warning: blobpacked storage detects non-zero packed zips, but no metadata. Please re-start in recovery mode.")
// TODO: add a recovery mode.
// Old TODO was:
// fail with a "known corrupt" message and refuse to
// start unless in recovery mode (perhaps a new environment
// var? or flag passed down?) using StreamBlobs starting at
// "l:". Could even do it automatically if total size is
// small or fast enough? But that's confusing if it only
// sometimes finishes recovery. We probably want various
// server start-up modes anyway: "check", "recover", "garbage
// collect", "readonly". So might as well introduce that
// concept now.
// TODO: test start-up recovery mode, once it works.
}
return sto, nil
}
// newStorage returns a new storage in path root with the given maxFileSize,
// or defaultMaxFileSize (512MB) if <= 0
func newStorage(root string, maxFileSize int64, indexConf jsonconfig.Obj) (s *storage, err error) {
fi, err := os.Stat(root)
if os.IsNotExist(err) {
return nil, fmt.Errorf("storage root %q doesn't exist", root)
}
if err != nil {
return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err)
}
if !fi.IsDir() {
return nil, fmt.Errorf("storage root %q exists but is not a directory.", root)
}
var index sorted.KeyValue
if len(indexConf) > 0 {
index, err = sorted.NewKeyValue(indexConf)
} else {
index, err = kvfile.NewStorage(filepath.Join(root, indexKV))
}
if err != nil {
return nil, err
}
defer func() {
if err != nil {
index.Close()
}
}()
if maxFileSize <= 0 {
maxFileSize = defaultMaxFileSize
}
// Be consistent with trailing slashes. Makes expvar stats for total
// reads/writes consistent across diskpacked targets, regardless of what
// people put in their low level config.
root = strings.TrimRight(root, `\/`)
s = &storage{
root: root,
index: index,
maxFileSize: maxFileSize,
Generationer: local.NewGenerationer(root),
}
s.mu.Lock()
defer s.mu.Unlock()
if err := s.openAllPacks(); err != nil {
return nil, err
}
if _, _, err := s.StorageGeneration(); err != nil {
return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err)
}
return s, nil
}
// TestMongoKV tests against a real MongoDB instance, using a Docker container.
func TestMongoKV(t *testing.T) {
// SetupMongoContainer may skip or fatal the test if docker isn't found or something goes wrong when setting up the container.
// Thus, no error is returned
containerID, ip := dockertest.SetupMongoContainer(t)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mongo",
"host": ip,
"database": "camlitest",
})
if err != nil {
t.Fatalf("mongo.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func TestKvfileKV(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "camlistore-kvfilekv_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
dbname := filepath.Join(tmpDir, "testdb.kvfile")
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "kv",
"file": dbname,
})
if err != nil {
t.Fatalf("Could not create kvfile sorted kv at %v: %v", dbname, err)
}
kvtest.TestSorted(t, kv)
}
func TestLeveldbKV(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "camlistore-leveldbkv_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
dbname := filepath.Join(tmpDir, "testdb.leveldb")
t.Logf("Testing leveldb %q.", dbname)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "leveldb",
"file": dbname,
})
if err != nil {
t.Fatalf("Could not create leveldb sorted kv at %v: %v", dbname, err)
}
kvtest.TestSorted(t, kv)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
sto := &nsto{}
invConf := config.RequiredObject("inventory")
masterName := config.RequiredString("storage")
if err := config.Validate(); err != nil {
return nil, err
}
sto.inventory, err = sorted.NewKeyValue(invConf)
if err != nil {
return nil, fmt.Errorf("Invalid 'inventory' configuration: %v", err)
}
sto.master, err = ld.GetStorage(masterName)
if err != nil {
return nil, fmt.Errorf("Invalid 'storage' configuration: %v", err)
}
return sto, nil
}
// TestMySQLKV tests against a real MySQL instance, using a Docker container.
func TestMySQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func newMongoSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMongoContainer(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mongo",
"host": ip,
"database": dbname,
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
// TestPostgreSQLKV tests against a real PostgreSQL instance, using a Docker container.
func TestPostgreSQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "postgres",
"host": ip,
"database": dbname,
"user": dockertest.PostgresUsername,
"password": dockertest.PostgresPassword,
"sslmode": "disable",
})
if err != nil {
t.Fatalf("postgres.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
f, err := ioutil.TempFile("", "sqlite-test")
if err != nil {
t.Fatal(err)
}
kv, err = sorted.NewKeyValue(jsonconfig.Obj{
"type": "sqlite",
"file": f.Name(),
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
os.Remove(f.Name())
}
}
func TestSQLiteKV(t *testing.T) {
if !CompiledIn() {
t.Skip(ErrNotCompiled.Error())
}
tmpDir, err := ioutil.TempDir("", "camlistore-sqlitekv_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
dbname := filepath.Join(tmpDir, "testdb.sqlite")
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "sqlite",
"file": dbname,
})
if err != nil {
t.Fatalf("Could not create sqlite sorted kv at %v: %v", dbname, err)
}
kvtest.TestSorted(t, kv)
}
func newMySQLSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
// TestMySQLKV tests against a real MySQL instance, using a Docker container.
func TestMySQLKV(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
// TODO(mpl): add test for serverVersion once we host the docker image ourselves
// (and hence have the control over the version).
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
kvtest.TestSorted(t, kv)
}
func newPostgresSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "postgres",
"host": ip,
"database": dbname,
"user": dockertest.PostgresUsername,
"password": dockertest.PostgresPassword,
"sslmode": "disable",
})
if err != nil {
containerID.KillRemove(t)
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.KillRemove(t)
}
}
func TestRollback(t *testing.T) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
defer containerID.KillRemove(t)
kv, err := sorted.NewKeyValue(jsonconfig.Obj{
"type": "mysql",
"host": ip + ":3306",
"database": dbname,
"user": dockertest.MySQLUsername,
"password": dockertest.MySQLPassword,
})
if err != nil {
t.Fatalf("mysql.NewKeyValue = %v", err)
}
tooLargeAKey := make([]byte, sorted.MaxKeySize+10)
for i := range tooLargeAKey {
tooLargeAKey[i] = 'L'
}
nbConnections := 2
tick := time.AfterFunc(5*time.Second, func() {
// We have to force close the connection, otherwise the connection hogging does not even
// let us exit the func with t.Fatal (How? why?)
kv.(*keyValue).DB.Close()
t.Fatal("Test failed because SQL connections blocked by unrolled transactions")
})
kv.(*keyValue).DB.SetMaxOpenConns(nbConnections)
for i := 0; i < nbConnections+1; i++ {
b := kv.BeginBatch()
// Making the transaction fail, to force a rollback
// -> this whole test fails before we introduce the rollback in CommitBatch.
b.Set(string(tooLargeAKey), "whatever")
if err := kv.CommitBatch(b); err == nil {
t.Fatal("wanted failed commit because too large a key")
}
}
tick.Stop()
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (bs blobserver.Storage, err error) {
metaConf := config.RequiredObject("metaIndex")
sto := &storage{}
agreement := config.OptionalString("I_AGREE", "")
const wantAgreement = "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change."
if agreement != wantAgreement {
return nil, errors.New("Use of the 'encrypt' target without the proper I_AGREE value.")
}
key := config.OptionalString("key", "")
keyFile := config.OptionalString("keyFile", "")
var keyb []byte
switch {
case key != "":
keyb, err = hex.DecodeString(key)
if err != nil || len(keyb) != 16 {
return nil, fmt.Errorf("The 'key' parameter must be 16 bytes of 32 hex digits. (currently fixed at AES-128)")
}
case keyFile != "":
// TODO: check that keyFile's unix permissions aren't too permissive.
keyb, err = ioutil.ReadFile(keyFile)
if err != nil {
return nil, fmt.Errorf("Reading key file %v: %v", keyFile, err)
}
}
blobStorage := config.RequiredString("blobs")
metaStorage := config.RequiredString("meta")
if err := config.Validate(); err != nil {
return nil, err
}
sto.index, err = sorted.NewKeyValue(metaConf)
if err != nil {
return
}
sto.blobs, err = ld.GetStorage(blobStorage)
if err != nil {
return
}
sto.meta, err = ld.GetStorage(metaStorage)
if err != nil {
return
}
if keyb == nil {
// TODO: add a way to prompt from stdin on start? or keychain support?
return nil, errors.New("no encryption key set with 'key' or 'keyFile'")
}
if err := sto.setKey(keyb); err != nil {
return nil, err
}
start := time.Now()
log.Printf("Reading encryption metadata...")
if err := sto.readAllMetaBlobs(); err != nil {
return nil, fmt.Errorf("Error scanning metadata on start-up: %v", err)
}
log.Printf("Read all encryption metadata in %.3f seconds", time.Since(start).Seconds())
return sto, nil
}
func newPublishHandler(conf *config) *publishHandler {
cl, err := app.Client()
if err != nil {
logger.Fatalf("could not get a client for the publish handler %v", err)
}
if conf.RootName == "" {
logger.Fatal("camliRoot not found in the app configuration")
}
maxResizeBytes := conf.MaxResizeBytes
if maxResizeBytes == 0 {
maxResizeBytes = constants.DefaultMaxResizeMem
}
var CSSFiles, JSDeps []string
if conf.SourceRoot != "" {
appRoot := filepath.Join(conf.SourceRoot, "app", "publisher")
Files = &fileembed.Files{
DirFallback: appRoot,
}
// TODO(mpl): Can I readdir by listing with "/" on Files, even with DirFallBack?
// Apparently not, but retry later.
dir, err := os.Open(appRoot)
if err != nil {
logger.Fatal(err)
}
defer dir.Close()
names, err := dir.Readdirnames(-1)
if err != nil {
logger.Fatal(err)
}
for _, v := range names {
if strings.HasSuffix(v, ".css") {
CSSFiles = append(CSSFiles, v)
continue
}
// TODO(mpl): document or fix (use a map?) the ordering
// problem: i.e. jquery.js must be sourced before
// publisher.js. For now, just cheat by sorting the
// slice.
if strings.HasSuffix(v, ".js") {
JSDeps = append(JSDeps, v)
}
}
sort.Strings(JSDeps)
} else {
Files.Listable = true
dir, err := Files.Open("/")
if err != nil {
logger.Fatal(err)
}
defer dir.Close()
fis, err := dir.Readdir(-1)
if err != nil {
logger.Fatal(err)
}
for _, v := range fis {
name := v.Name()
if strings.HasSuffix(name, ".css") {
CSSFiles = append(CSSFiles, name)
continue
}
if strings.HasSuffix(name, ".js") {
JSDeps = append(JSDeps, name)
}
}
sort.Strings(JSDeps)
}
// TODO(mpl): add all htmls found in Files to the template if none specified?
if conf.GoTemplate == "" {
logger.Fatal("a go template is required in the app configuration")
}
goTemplate, err := goTemplate(Files, conf.GoTemplate)
if err != nil {
logger.Fatal(err)
}
var cache blobserver.Storage
var thumbMeta *server.ThumbMeta
if conf.CacheRoot != "" {
cache, err = localdisk.New(conf.CacheRoot)
if err != nil {
logger.Fatalf("Could not create localdisk cache: %v", err)
}
thumbsCacheDir := filepath.Join(os.TempDir(), "camli-publisher-cache")
if err := os.MkdirAll(thumbsCacheDir, 0700); err != nil {
logger.Fatalf("Could not create cache dir %s for %v publisher: %v", thumbsCacheDir, conf.RootName, err)
}
kv, err := sorted.NewKeyValue(map[string]interface{}{
"type": "kv",
"file": filepath.Join(thumbsCacheDir, conf.RootName+"-thumbnails.kv"),
})
if err != nil {
logger.Fatalf("Could not create kv for %v's thumbs cache: %v", conf.RootName, err)
}
thumbMeta = server.NewThumbMeta(kv)
}
return &publishHandler{
rootName: conf.RootName,
cl: cl,
resizeSem: syncutil.NewSem(maxResizeBytes),
staticFiles: Files,
goTemplate: goTemplate,
CSSFiles: CSSFiles,
JSDeps: JSDeps,
describedCache: make(map[string]*search.DescribedBlob),
cache: cache,
thumbMeta: thumbMeta,
}
}
// newKVOrNil wraps sorted.NewKeyValue and adds the ability
// to pass a nil conf to get a (nil, nil) response.
func newKVOrNil(conf jsonconfig.Obj) (sorted.KeyValue, error) {
if len(conf) == 0 {
return nil, nil
}
return sorted.NewKeyValue(conf)
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
from = conf.RequiredString("from")
to = conf.RequiredString("to")
fullSync = conf.OptionalBool("fullSyncOnStart", false)
blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false)
idle = conf.OptionalBool("idle", false)
queueConf = conf.OptionalObject("queue")
copierPoolSize = conf.OptionalInt("copierPoolSize", 5)
validate = conf.OptionalBool("validateOnStart", validateOnStartDefault)
)
if err := conf.Validate(); err != nil {
return nil, err
}
if idle {
return newIdleSyncHandler(from, to), nil
}
if len(queueConf) == 0 {
return nil, errors.New(`Missing required "queue" object`)
}
q, err := sorted.NewKeyValue(queueConf)
if err != nil {
return nil, err
}
isToIndex := false
fromBs, err := ld.GetStorage(from)
if err != nil {
return nil, err
}
toBs, err := ld.GetStorage(to)
if err != nil {
return nil, err
}
if _, ok := fromBs.(*index.Index); !ok {
if _, ok := toBs.(*index.Index); ok {
isToIndex = true
}
}
sh := newSyncHandler(from, to, fromBs, toBs, q)
sh.toIndex = isToIndex
sh.copierPoolSize = copierPoolSize
if err := sh.readQueueToMemory(); err != nil {
return nil, fmt.Errorf("Error reading sync queue to memory: %v", err)
}
if fullSync || blockFullSync {
sh.logf("Doing full sync")
didFullSync := make(chan bool, 1)
go func() {
for {
n := sh.runSync("queue", sh.enumeratePendingBlobs)
if n > 0 {
sh.logf("Queue sync copied %d blobs", n)
continue
}
break
}
n := sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs))
sh.logf("Full sync copied %d blobs", n)
didFullSync <- true
sh.syncLoop()
}()
if blockFullSync {
sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
sh.logf("Full sync complete.")
}
} else {
go sh.syncLoop()
}
if validate {
go sh.startFullValidation()
}
blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue)
return sh, nil
}
func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
var (
from = conf.RequiredString("from")
to = conf.RequiredString("to")
fullSync = conf.OptionalBool("fullSyncOnStart", false)
blockFullSync = conf.OptionalBool("blockingFullSyncOnStart", false)
idle = conf.OptionalBool("idle", false)
queueConf = conf.OptionalObject("queue")
)
if err := conf.Validate(); err != nil {
return nil, err
}
if idle {
synch, err := createIdleSyncHandler(from, to)
if err != nil {
return nil, err
}
return synch, nil
}
if len(queueConf) == 0 {
return nil, errors.New(`Missing required "queue" object`)
}
q, err := sorted.NewKeyValue(queueConf)
if err != nil {
return nil, err
}
isToIndex := false
fromBs, err := ld.GetStorage(from)
if err != nil {
return nil, err
}
toBs, err := ld.GetStorage(to)
if err != nil {
return nil, err
}
if _, ok := fromBs.(*index.Index); !ok {
if _, ok := toBs.(*index.Index); ok {
isToIndex = true
}
}
sh, err := createSyncHandler(from, to, fromBs, toBs, q, isToIndex)
if err != nil {
return nil, err
}
if fullSync || blockFullSync {
didFullSync := make(chan bool, 1)
go func() {
n := sh.runSync("queue", sh.enumerateQueuedBlobs)
sh.logf("Queue sync copied %d blobs", n)
n = sh.runSync("full", blobserverEnumerator(context.TODO(), fromBs))
sh.logf("Full sync copied %d blobs", n)
didFullSync <- true
sh.syncQueueLoop()
}()
if blockFullSync {
sh.logf("Blocking startup, waiting for full sync from %q to %q", from, to)
<-didFullSync
sh.logf("Full sync complete.")
}
} else {
go sh.syncQueueLoop()
}
blobserver.GetHub(fromBs).AddReceiveHook(sh.enqueue)
return sh, nil
}