func TestSS_SaveRestore(t *testing.T) {
	loc := datastore.NewINodeDBSSLocator(authtu.TestDSConfig(testRootKey()))
	if _, err := loc.DeleteAll(); err != nil {
		t.Errorf("Failed to loc.DeleteAll: %v", err)
	}

	bs := tu.TestFileBlobStore()
	sio := blobstoredbstatesnapshotio.New(bs, tu.TestCipher(), loc)

	db, err := inodedb.NewEmptyDB(sio, inodedb.NewSimpleDBTransactionLogIO())
	if err != nil {
		t.Errorf("Failed to NewEmptyDB: %v", err)
		return
	}
	if err := db.Sync(); err != nil {
		t.Errorf("Failed to Sync DB: %v", err)
		return
	}

	_, err = inodedb.NewDB(sio, inodedb.NewSimpleDBTransactionLogIO())
	if err != nil {
		t.Errorf("Failed to NewDB: %v", err)
	}
}
Exemplo n.º 2
0
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()

	if !cfg.LocalDebug {
		o.Clisrc, err = auth.GetGCloudClientSource(
			path.Join(os.Getenv("HOME"), ".otaru", "credentials.json"),
			path.Join(os.Getenv("HOME"), ".otaru", "tokencache.json"),
			false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Clisrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Clisrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)

	o.SIO = otaru.NewBlobStoreDBStateSnapshotIO(o.CBS, o.C)

	if !cfg.LocalDebug {
		o.TxIO, err = datastore.NewDBTransactionLogIO(cfg.ProjectName, cfg.BucketName, o.C, o.Clisrc)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
		err = nil
	}
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init gcloud DBTransactionLogIO: %v", err)
	}

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.TxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.TxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
	o.MGMT = mgmt.NewServer()
	o.setupMgmtAPIs()
	if err := o.runMgmtServer(); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}
func TestSS_AutoAvoidCorruptedSnapshot(t *testing.T) {
	loc := datastore.NewINodeDBSSLocator(authtu.TestDSConfig(testRootKey()))
	if _, err := loc.DeleteAll(); err != nil {
		t.Errorf("Failed to loc.DeleteAll: %v", err)
	}

	bs := tu.TestFileBlobStore()
	sio := blobstoredbstatesnapshotio.New(bs, tu.TestCipher(), loc)
	txlogio := inodedb.NewSimpleDBTransactionLogIO()

	{
		db, err := inodedb.NewEmptyDB(sio, txlogio)
		if err != nil {
			t.Errorf("Failed to NewEmptyDB: %v", err)
			return
		}

		// create 1st snapshot
		if err := db.Sync(); err != nil {
			t.Errorf("Failed to Sync DB: %v", err)
			return
		}

		// apply some mod to inodedb
		nlock, err := db.LockNode(inodedb.AllocateNewNodeID)
		if err != nil {
			t.Errorf("Failed to LockNode: %v", err)
			return
		}

		tx := inodedb.DBTransaction{Ops: []inodedb.DBOperation{
			&inodedb.CreateNodeOp{NodeLock: nlock, OrigPath: "/hoge.txt", Type: inodedb.FileNodeT},
			&inodedb.HardLinkOp{NodeLock: inodedb.NodeLock{1, inodedb.NoTicket}, Name: "hoge.txt", TargetID: nlock.ID},
		}}
		if _, err := db.ApplyTransaction(tx); err != nil {
			t.Errorf("Failed to apply tx: %v", err)
			return
		}

		if err := db.UnlockNode(nlock); err != nil {
			t.Errorf("Failed to UnlockNode: %v", err)
			return
		}

		// create 2nd snapshot
		if err := db.Sync(); err != nil {
			t.Errorf("Failed to Sync DB (2): %v", err)
			return
		}
	}

	if _, err := inodedb.NewDB(sio, txlogio); err != nil {
		t.Errorf("Failed to NewDB (uncorrupted): %v", err)
		return
	}

	// destroy latest snapshot (corrupt data)
	ssbp, err := loc.Locate(0)
	if err != nil {
		t.Errorf("Failed to locate latest ssbp: %v", err)
		return
	}
	{
		wc, err := bs.OpenWriter(ssbp)
		if err != nil {
			t.Errorf("Failed to OpenWriter: %v", err)
			return
		}
		if _, err := wc.Write([]byte("hoge")); err != nil {
			t.Errorf("Failed to Write: %v", err)
		}
		wc.Close()
	}

	{
		_, err = inodedb.NewDB(sio, txlogio)
		if err != nil {
			t.Errorf("Failed to NewDB (corrupted): %v", err)
			return
		}
	}

	// destroy latest snapshot (remove ss blob)
	if err := bs.RemoveBlob(ssbp); err != nil {
		t.Errorf("Failed to RemoveBlob: %v", err)
	}

	{
		_, err = inodedb.NewDB(sio, txlogio)
		if err != nil {
			t.Errorf("Failed to NewDB (ss blob removed): %v", err)
			return
		}
	}
}
Exemplo n.º 4
0
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()

	if !cfg.LocalDebug {
		o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
		o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
		o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
		if err := o.GL.Lock(); err != nil {
			return nil, err
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	if err := o.CBS.RestoreState(o.C); err != nil {
		logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
	}
	o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)

	if !cfg.LocalDebug {
		o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
	} else {
		logger.Panicf(mylog, "Implement mock sslocator that doesn't depend on gcloud/datastore")
	}
	o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)

	if !cfg.LocalDebug {
		txio := datastore.NewDBTransactionLogIO(o.DSCfg)
		o.TxIO = txio
		o.TxIOSS = util.NewSyncScheduler(txio, 300*time.Millisecond)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
	}
	o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
	o.MGMT = mgmt.NewServer()
	if err := o.runMgmtServer(); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}