func TestCachedBlobStore_AutoExpandLen(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
bs, err := cachedblobstore.New(backendbs, cachebs, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
bh, err := bs.Open("hoge", flags.O_RDWRCREATE)
if err != nil {
t.Errorf("Failed to open blobhandle")
}
defer bh.Close()
if size := bh.Size(); size != 0 {
t.Errorf("New bh size non-zero: %d", size)
}
if err := bh.PWrite(0, []byte("Hello")); err != nil {
t.Errorf("PWrite failed: %v", err)
}
if size := bh.Size(); size != 5 {
t.Errorf("bh size not auto expanded! size %d", size)
}
}
func TestCachedBlobStore_NewEntry(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
bs, err := cachedblobstore.New(backendbs, cachebs, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "newentry", 1); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersionRA(bs, "newentry", 1); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.Sync(); err != nil {
t.Errorf("Sync failed: %v", err)
return
}
if err := tu.AssertBlobVersion(cachebs, "newentry", 1); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersion(backendbs, "newentry", 1); err != nil {
t.Errorf("%v", err)
return
}
}
func TestCachedBlobStore(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
s := scheduler.NewScheduler()
if err := tu.WriteVersionedBlob(backendbs, "backendonly", 5); err != nil {
t.Errorf("%v", err)
return
}
bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
if err := tu.AssertBlobVersion(backendbs, "backendonly", 5); err != nil {
t.Errorf("%v", err)
return
}
// assert cache not yet filled
if err := tu.AssertBlobVersion(cachebs, "backendonly", 0); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersionRA(bs, "backendonly", 5); err != nil {
t.Errorf("%v", err)
return
}
// assert cache fill
if err := tu.AssertBlobVersion(cachebs, "backendonly", 5); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "backendonly", 10); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.Sync(); err != nil {
t.Errorf("Sync failed: %v", err)
return
}
if err := tu.AssertBlobVersionRA(bs, "backendonly", 10); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersion(cachebs, "backendonly", 10); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersion(backendbs, "backendonly", 10); err != nil {
t.Errorf("%v", err)
return
}
}
func TestCachedBlobStore_Invalidate(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
if err := tu.WriteVersionedBlob(cachebs, "backendnewer", 2); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlob(backendbs, "backendnewer", 3); err != nil {
t.Errorf("%v", err)
return
}
bs, err := cachedblobstore.New(backendbs, cachebs, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
if err := tu.AssertBlobVersionRA(bs, "backendnewer", 3); err != nil {
t.Errorf("%v", err)
return
}
// assert cache fill
if err := tu.AssertBlobVersion(cachebs, "backendnewer", 3); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "backendnewer", 4); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersionRA(bs, "backendnewer", 4); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.Sync(); err != nil {
t.Errorf("Sync failed: %v", err)
return
}
if err := tu.AssertBlobVersion(cachebs, "backendnewer", 4); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.AssertBlobVersion(backendbs, "backendnewer", 4); err != nil {
t.Errorf("%v", err)
return
}
}
func TestCachedBlobStore_ListBlobs(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
s := scheduler.NewScheduler()
bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
if err := tu.WriteVersionedBlob(backendbs, "backendonly", 1); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlob(cachebs, "cacheonly", 2); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "synced", 3); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.Sync(); err != nil {
t.Errorf("Sync failed: %v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "unsynced", 4); err != nil {
t.Errorf("%v", err)
return
}
bpaths, err := bs.ListBlobs()
if err != nil {
t.Errorf("ListBlobs failed: %v", err)
return
}
sort.Strings(bpaths)
if !reflect.DeepEqual([]string{"backendonly", "synced", "unsynced"}, bpaths) {
t.Errorf("ListBlobs returned unexpected result: %v", bpaths)
}
}
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
o := &Otaru{}
var err error
key := btncrypt.KeyFromPassword(cfg.Password)
o.C, err = btncrypt.NewCipher(key)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init Cipher: %v", err)
}
o.S = scheduler.NewScheduler()
if !cfg.LocalDebug {
o.Clisrc, err = auth.GetGCloudClientSource(
path.Join(os.Getenv("HOME"), ".otaru", "credentials.json"),
path.Join(os.Getenv("HOME"), ".otaru", "tokencache.json"),
false)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
}
}
o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
}
if !cfg.LocalDebug {
o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Clisrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
}
if !cfg.UseSeparateBucketForMetadata {
o.BackendBS = o.DefaultBS
} else {
metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Clisrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
}
o.BackendBS = blobstore.Mux{
blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
blobstore.MuxEntry{nil, o.DefaultBS},
}
}
} else {
o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
}
queryFn := chunkstore.NewQueryChunkVersion(o.C)
o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, oflags.O_RDWRCREATE /* FIXME */, queryFn)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
}
o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)
o.SIO = otaru.NewBlobStoreDBStateSnapshotIO(o.CBS, o.C)
if !cfg.LocalDebug {
o.TxIO, err = datastore.NewDBTransactionLogIO(cfg.ProjectName, cfg.BucketName, o.C, o.Clisrc)
} else {
o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
err = nil
}
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init gcloud DBTransactionLogIO: %v", err)
}
if oneshotcfg.Mkfs {
o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.TxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
}
} else {
o.IDBBE, err = inodedb.NewDB(o.SIO, o.TxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewDB failed: %v", err)
}
}
o.IDBS = inodedb.NewDBService(o.IDBBE)
o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)
o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
o.MGMT = mgmt.NewServer()
o.setupMgmtAPIs()
if err := o.runMgmtServer(); err != nil {
o.Close()
return nil, fmt.Errorf("Mgmt server run failed: %v", err)
}
return o, nil
}
func TestCachedBlobStore_RemoveBlob(t *testing.T) {
backendbs := tu.TestFileBlobStoreOfName("backend")
cachebs := tu.TestFileBlobStoreOfName("cache")
bs, err := cachedblobstore.New(backendbs, cachebs, flags.O_RDWRCREATE, tu.TestQueryVersion)
if err != nil {
t.Errorf("Failed to create CachedBlobStore: %v", err)
return
}
if err := tu.WriteVersionedBlob(backendbs, "backendonly", 1); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlob(cachebs, "cacheonly", 2); err != nil {
t.Errorf("%v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "synced", 3); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.Sync(); err != nil {
t.Errorf("Sync failed: %v", err)
return
}
if err := tu.WriteVersionedBlobRA(bs, "unsynced", 4); err != nil {
t.Errorf("%v", err)
return
}
if err := bs.RemoveBlob("backendonly"); err != nil {
t.Errorf("RemoveBlob failed: %v", err)
return
}
if err := bs.RemoveBlob("synced"); err != nil {
t.Errorf("RemoveBlob failed: %v", err)
return
}
if err := bs.RemoveBlob("unsynced"); err != nil {
t.Errorf("RemoveBlob failed: %v", err)
return
}
bpaths, err := bs.ListBlobs()
if err != nil {
t.Errorf("ListBlobs failed: %v", err)
return
}
if len(bpaths) > 0 {
t.Errorf("Left over blobs: %v", bpaths)
}
for _, bp := range []string{"backendonly", "synced", "unsynced"} {
if err := tu.AssertBlobVersionRA(bs, bp, 0); err != nil {
t.Errorf("left over blob in bs: %s", bp)
}
if err := tu.AssertBlobVersion(cachebs, bp, 0); err != nil {
t.Errorf("left over blob in cachebs: %s", bp)
}
if err := tu.AssertBlobVersion(backendbs, bp, 0); err != nil {
t.Errorf("left over blob in backendbs: %s", bp)
}
}
}
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
o := &Otaru{}
var err error
key := btncrypt.KeyFromPassword(cfg.Password)
o.C, err = btncrypt.NewCipher(key)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init Cipher: %v", err)
}
o.S = scheduler.NewScheduler()
if !cfg.LocalDebug {
o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
}
o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
if err := o.GL.Lock(); err != nil {
return nil, err
}
}
o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
}
if !cfg.LocalDebug {
o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
}
if !cfg.UseSeparateBucketForMetadata {
o.BackendBS = o.DefaultBS
} else {
metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
}
o.BackendBS = blobstore.Mux{
blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
blobstore.MuxEntry{nil, o.DefaultBS},
}
}
} else {
o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
}
queryFn := chunkstore.NewQueryChunkVersion(o.C)
o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
}
if err := o.CBS.RestoreState(o.C); err != nil {
logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
}
o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)
if !cfg.LocalDebug {
o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
} else {
logger.Panicf(mylog, "Implement mock sslocator that doesn't depend on gcloud/datastore")
}
o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)
if !cfg.LocalDebug {
txio := datastore.NewDBTransactionLogIO(o.DSCfg)
o.TxIO = txio
o.TxIOSS = util.NewSyncScheduler(txio, 300*time.Millisecond)
} else {
o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
}
o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)
if oneshotcfg.Mkfs {
o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
}
} else {
o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewDB failed: %v", err)
}
}
o.IDBS = inodedb.NewDBService(o.IDBBE)
o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)
o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
o.MGMT = mgmt.NewServer()
if err := o.runMgmtServer(); err != nil {
o.Close()
return nil, fmt.Errorf("Mgmt server run failed: %v", err)
}
return o, nil
}