func TestCreateFile(t *testing.T) {
db, err := i.NewEmptyDB(i.NewSimpleDBStateSnapshotIO(), i.NewSimpleDBTransactionLogIO())
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
nlock, err := db.LockNode(i.AllocateNewNodeID)
if err != nil {
t.Errorf("Failed to LockNode: %v", err)
return
}
tx := i.DBTransaction{Ops: []i.DBOperation{
&i.CreateNodeOp{NodeLock: nlock, OrigPath: "/hoge.txt", Type: i.FileNodeT},
&i.HardLinkOp{NodeLock: i.NodeLock{1, i.NoTicket}, Name: "hoge.txt", TargetID: nlock.ID},
}}
if _, err := db.ApplyTransaction(tx); err != nil {
t.Errorf("Failed to apply tx: %v", err)
}
if err := db.UnlockNode(nlock); err != nil {
t.Errorf("Failed to UnlockNode: %v", err)
}
fbps, errs := db.Fsck()
if len(fbps) != 0 {
t.Errorf("Fsck returned used fbp on db: %v", fbps)
}
if len(errs) != 0 {
t.Errorf("Fsck returned err on db: %v", errs)
}
}
func TestFileWriteRead(t *testing.T) {
snapshotio := inodedb.NewSimpleDBStateSnapshotIO()
txio := inodedb.NewSimpleDBTransactionLogIO()
idb, err := inodedb.NewEmptyDB(snapshotio, txio)
if err != nil {
t.Errorf("NewEmptyDB failed: %v", err)
return
}
bs := tu.TestFileBlobStore()
fs := otaru.NewFileSystem(idb, bs, tu.TestCipher())
h, err := fs.OpenFileFullPath("/hello.txt", flags.O_CREATE|flags.O_RDWR, 0666)
if err != nil {
t.Errorf("OpenFileFullPath failed: %v", err)
return
}
err = h.PWrite(tu.HelloWorld, 0)
if err != nil {
t.Errorf("PWrite failed: %v", err)
}
buf := make([]byte, 32)
n, err := h.ReadAt(buf, 0)
if err != nil {
t.Errorf("PRead failed: %v", err)
}
buf = buf[:n]
if n != len(tu.HelloWorld) {
t.Errorf("n: %d", n)
}
if !bytes.Equal(tu.HelloWorld, buf) {
t.Errorf("PRead content != PWrite content: %v", buf)
}
}
func TestINodeDB_Rollback(t *testing.T) {
sio := i.NewSimpleDBStateSnapshotIO()
txio := i.NewSimpleDBTransactionLogIO()
db, err := i.NewEmptyDB(sio, txio)
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
tx := i.DBTransaction{Ops: []i.DBOperation{
&i.CreateNodeOp{NodeLock: i.NodeLock{2, 123456}, OrigPath: "/hoge.txt", Type: i.FileNodeT},
&i.HardLinkOp{NodeLock: i.NodeLock{1, i.NoTicket}, Name: "hoge.txt", TargetID: 2},
}}
if _, err := db.ApplyTransaction(tx); err != nil {
t.Errorf("ApplyTransaction failed: %v", err)
return
}
txF := i.DBTransaction{Ops: []i.DBOperation{
&i.AlwaysFailForTestingOp{},
}}
if _, err := db.ApplyTransaction(txF); err == nil {
t.Errorf("ApplyTransaction succeeded unexpectedly!")
}
}
func TestFileWriteRead(t *testing.T) {
snapshotio := inodedb.NewSimpleDBStateSnapshotIO()
txio := inodedb.NewSimpleDBTransactionLogIO()
idb, err := inodedb.NewEmptyDB(snapshotio, txio)
if err != nil {
t.Errorf("NewEmptyDB failed: %v", err)
return
}
bs := TestFileBlobStore()
fs := otaru.NewFileSystem(idb, bs, TestCipher())
h, err := fs.OpenFileFullPath("/hello.txt", flags.O_CREATE|flags.O_RDWR, 0666)
if err != nil {
t.Errorf("OpenFileFullPath failed: %v", err)
return
}
err = h.PWrite(0, []byte("hello world!\n"))
if err != nil {
t.Errorf("PWrite failed: %v", err)
}
buf := make([]byte, 13)
err = h.PRead(0, buf)
if err != nil {
t.Errorf("PRead failed: %v", err)
}
if !bytes.Equal([]byte("hello world!\n"), buf) {
t.Errorf("PRead content != PWrite content")
}
}
func TestNewEmptyDB_ShouldFailOnNonEmptySnapsshotIO(t *testing.T) {
sio := i.NewSimpleDBStateSnapshotIO()
{
db, err := i.NewEmptyDB(sio, i.NewSimpleDBTransactionLogIO())
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
if err := db.Sync(); err != nil {
t.Errorf("Failed to Sync DB: %v", err)
return
}
}
_, err := i.NewEmptyDB(sio, i.NewSimpleDBTransactionLogIO())
if err == nil {
t.Errorf("NewEmptyDB should fail on non-empty snapshot io")
return
}
}
func fusetestFileSystem() *otaru.FileSystem {
sio := inodedb.NewSimpleDBStateSnapshotIO()
txio := inodedb.NewSimpleDBTransactionLogIO()
idb, err := inodedb.NewEmptyDB(sio, txio)
if err != nil {
log.Fatalf("NewEmptyDB failed: %v", err)
}
bs := TestFileBlobStore()
fs := otaru.NewFileSystem(idb, bs, TestCipher())
return fs
}
func TestNewEmptyDB_ShouldFailOnNonEmptyTxIO(t *testing.T) {
sio := i.NewSimpleDBStateSnapshotIO()
txio := i.NewSimpleDBTransactionLogIO()
tx := i.DBTransaction{TxID: 123, Ops: []i.DBOperation{
&i.CreateNodeOp{NodeLock: i.NodeLock{2, 123456}, OrigPath: "/hoge.txt", Type: i.FileNodeT},
&i.HardLinkOp{NodeLock: i.NodeLock{1, i.NoTicket}, Name: "hoge.txt", TargetID: 2},
}}
if err := txio.AppendTransaction(tx); err != nil {
t.Errorf("AppendTransaction failed: %v", err)
return
}
_, err := i.NewEmptyDB(sio, txio)
if err == nil {
t.Errorf("NewEmptyDB should fail on non-empty txio")
return
}
}
func TestSS_SaveRestore(t *testing.T) {
loc := datastore.NewINodeDBSSLocator(authtu.TestDSConfig(testRootKey()))
if _, err := loc.DeleteAll(); err != nil {
t.Errorf("Failed to loc.DeleteAll: %v", err)
}
bs := tu.TestFileBlobStore()
sio := blobstoredbstatesnapshotio.New(bs, tu.TestCipher(), loc)
db, err := inodedb.NewEmptyDB(sio, inodedb.NewSimpleDBTransactionLogIO())
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
if err := db.Sync(); err != nil {
t.Errorf("Failed to Sync DB: %v", err)
return
}
_, err = inodedb.NewDB(sio, inodedb.NewSimpleDBTransactionLogIO())
if err != nil {
t.Errorf("Failed to NewDB: %v", err)
}
}
func TestInitialState(t *testing.T) {
db, err := i.NewEmptyDB(i.NewSimpleDBStateSnapshotIO(), i.NewSimpleDBTransactionLogIO())
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
nv, _, err := db.QueryNode(1, false)
if err != nil {
t.Errorf("Failed to query root dir")
return
}
if nv.GetType() != i.DirNodeT {
t.Errorf("root dir not found!")
}
fbps, errs := db.Fsck()
if len(fbps) != 0 {
t.Errorf("Fsck returned used fbp on new empty db: %v", fbps)
}
if len(errs) != 0 {
t.Errorf("Fsck returned err on new empty db: %v", errs)
}
}
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
o := &Otaru{}
var err error
key := btncrypt.KeyFromPassword(cfg.Password)
o.C, err = btncrypt.NewCipher(key)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init Cipher: %v", err)
}
o.S = scheduler.NewScheduler()
if !cfg.LocalDebug {
o.Clisrc, err = auth.GetGCloudClientSource(
path.Join(os.Getenv("HOME"), ".otaru", "credentials.json"),
path.Join(os.Getenv("HOME"), ".otaru", "tokencache.json"),
false)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
}
}
o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
}
if !cfg.LocalDebug {
o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Clisrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
}
if !cfg.UseSeparateBucketForMetadata {
o.BackendBS = o.DefaultBS
} else {
metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Clisrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
}
o.BackendBS = blobstore.Mux{
blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
blobstore.MuxEntry{nil, o.DefaultBS},
}
}
} else {
o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
}
queryFn := chunkstore.NewQueryChunkVersion(o.C)
o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, oflags.O_RDWRCREATE /* FIXME */, queryFn)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
}
o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)
o.SIO = otaru.NewBlobStoreDBStateSnapshotIO(o.CBS, o.C)
if !cfg.LocalDebug {
o.TxIO, err = datastore.NewDBTransactionLogIO(cfg.ProjectName, cfg.BucketName, o.C, o.Clisrc)
} else {
o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
err = nil
}
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init gcloud DBTransactionLogIO: %v", err)
}
if oneshotcfg.Mkfs {
o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.TxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
}
} else {
o.IDBBE, err = inodedb.NewDB(o.SIO, o.TxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewDB failed: %v", err)
}
}
o.IDBS = inodedb.NewDBService(o.IDBBE)
o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)
o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
o.MGMT = mgmt.NewServer()
o.setupMgmtAPIs()
if err := o.runMgmtServer(); err != nil {
o.Close()
return nil, fmt.Errorf("Mgmt server run failed: %v", err)
}
return o, nil
}
func TestSS_AutoAvoidCorruptedSnapshot(t *testing.T) {
loc := datastore.NewINodeDBSSLocator(authtu.TestDSConfig(testRootKey()))
if _, err := loc.DeleteAll(); err != nil {
t.Errorf("Failed to loc.DeleteAll: %v", err)
}
bs := tu.TestFileBlobStore()
sio := blobstoredbstatesnapshotio.New(bs, tu.TestCipher(), loc)
txlogio := inodedb.NewSimpleDBTransactionLogIO()
{
db, err := inodedb.NewEmptyDB(sio, txlogio)
if err != nil {
t.Errorf("Failed to NewEmptyDB: %v", err)
return
}
// create 1st snapshot
if err := db.Sync(); err != nil {
t.Errorf("Failed to Sync DB: %v", err)
return
}
// apply some mod to inodedb
nlock, err := db.LockNode(inodedb.AllocateNewNodeID)
if err != nil {
t.Errorf("Failed to LockNode: %v", err)
return
}
tx := inodedb.DBTransaction{Ops: []inodedb.DBOperation{
&inodedb.CreateNodeOp{NodeLock: nlock, OrigPath: "/hoge.txt", Type: inodedb.FileNodeT},
&inodedb.HardLinkOp{NodeLock: inodedb.NodeLock{1, inodedb.NoTicket}, Name: "hoge.txt", TargetID: nlock.ID},
}}
if _, err := db.ApplyTransaction(tx); err != nil {
t.Errorf("Failed to apply tx: %v", err)
return
}
if err := db.UnlockNode(nlock); err != nil {
t.Errorf("Failed to UnlockNode: %v", err)
return
}
// create 2nd snapshot
if err := db.Sync(); err != nil {
t.Errorf("Failed to Sync DB (2): %v", err)
return
}
}
if _, err := inodedb.NewDB(sio, txlogio); err != nil {
t.Errorf("Failed to NewDB (uncorrupted): %v", err)
return
}
// destroy latest snapshot (corrupt data)
ssbp, err := loc.Locate(0)
if err != nil {
t.Errorf("Failed to locate latest ssbp: %v", err)
return
}
{
wc, err := bs.OpenWriter(ssbp)
if err != nil {
t.Errorf("Failed to OpenWriter: %v", err)
return
}
if _, err := wc.Write([]byte("hoge")); err != nil {
t.Errorf("Failed to Write: %v", err)
}
wc.Close()
}
{
_, err = inodedb.NewDB(sio, txlogio)
if err != nil {
t.Errorf("Failed to NewDB (corrupted): %v", err)
return
}
}
// destroy latest snapshot (remove ss blob)
if err := bs.RemoveBlob(ssbp); err != nil {
t.Errorf("Failed to RemoveBlob: %v", err)
}
{
_, err = inodedb.NewDB(sio, txlogio)
if err != nil {
t.Errorf("Failed to NewDB (ss blob removed): %v", err)
return
}
}
}
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
o := &Otaru{}
var err error
key := btncrypt.KeyFromPassword(cfg.Password)
o.C, err = btncrypt.NewCipher(key)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init Cipher: %v", err)
}
o.S = scheduler.NewScheduler()
if !cfg.LocalDebug {
o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
}
o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
if err := o.GL.Lock(); err != nil {
return nil, err
}
}
o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
}
if !cfg.LocalDebug {
o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
}
if !cfg.UseSeparateBucketForMetadata {
o.BackendBS = o.DefaultBS
} else {
metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
}
o.BackendBS = blobstore.Mux{
blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
blobstore.MuxEntry{nil, o.DefaultBS},
}
}
} else {
o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
}
queryFn := chunkstore.NewQueryChunkVersion(o.C)
o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
if err != nil {
o.Close()
return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
}
if err := o.CBS.RestoreState(o.C); err != nil {
logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
}
o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)
if !cfg.LocalDebug {
o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
} else {
logger.Panicf(mylog, "Implement mock sslocator that doesn't depend on gcloud/datastore")
}
o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)
if !cfg.LocalDebug {
txio := datastore.NewDBTransactionLogIO(o.DSCfg)
o.TxIO = txio
o.TxIOSS = util.NewSyncScheduler(txio, 300*time.Millisecond)
} else {
o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
}
o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)
if oneshotcfg.Mkfs {
o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
}
} else {
o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
if err != nil {
o.Close()
return nil, fmt.Errorf("NewDB failed: %v", err)
}
}
o.IDBS = inodedb.NewDBService(o.IDBBE)
o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)
o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
o.MGMT = mgmt.NewServer()
if err := o.runMgmtServer(); err != nil {
o.Close()
return nil, fmt.Errorf("Mgmt server run failed: %v", err)
}
return o, nil
}