Exemplo n.º 1
0
func TestCachedBlobStore_AutoExpandLen(t *testing.T) {
	backendbs := tu.TestFileBlobStoreOfName("backend")
	cachebs := tu.TestFileBlobStoreOfName("cache")
	s := scheduler.NewScheduler()

	bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
	if err != nil {
		t.Errorf("Failed to create CachedBlobStore: %v", err)
		return
	}

	bh, err := bs.Open("hoge", flags.O_RDWRCREATE)
	if err != nil {
		t.Errorf("Failed to open blobhandle")
	}
	defer bh.Close()

	if size := bh.Size(); size != 0 {
		t.Errorf("New bh size non-zero: %d", size)
	}

	if err := bh.PWrite([]byte("Hello"), 0); err != nil {
		t.Errorf("PWrite failed: %v", err)
	}

	if size := bh.Size(); size != 5 {
		t.Errorf("bh size not auto expanded! size  %d", size)
	}
}
Exemplo n.º 2
0
func TestCachedBlobStore_NewEntry(t *testing.T) {
	backendbs := tu.TestFileBlobStoreOfName("backend")
	cachebs := tu.TestFileBlobStoreOfName("cache")
	s := scheduler.NewScheduler()

	bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
	if err != nil {
		t.Errorf("Failed to create CachedBlobStore: %v", err)
		return
	}

	if err := tu.WriteVersionedBlobRA(bs, "newentry", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.AssertBlobVersionRA(bs, "newentry", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := bs.Sync(); err != nil {
		t.Errorf("Sync failed: %v", err)
		return
	}
	if err := tu.AssertBlobVersion(cachebs, "newentry", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.AssertBlobVersion(backendbs, "newentry", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
}
Exemplo n.º 3
0
func TestCachedBlobStore(t *testing.T) {
	backendbs := tu.TestFileBlobStoreOfName("backend")
	cachebs := tu.TestFileBlobStoreOfName("cache")
	s := scheduler.NewScheduler()

	if err := tu.WriteVersionedBlob(backendbs, "backendonly", 5); err != nil {
		t.Errorf("%v", err)
		return
	}

	bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
	if err != nil {
		t.Errorf("Failed to create CachedBlobStore: %v", err)
		return
	}
	if err := tu.AssertBlobVersion(backendbs, "backendonly", 5); err != nil {
		t.Errorf("%v", err)
		return
	}
	// assert cache not yet filled
	if err := tu.AssertBlobVersion(cachebs, "backendonly", 0); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.AssertBlobVersionRA(bs, "backendonly", 5); err != nil {
		t.Errorf("%v", err)
		return
	}

	// assert cache fill
	if err := tu.AssertBlobVersion(cachebs, "backendonly", 5); err != nil {
		t.Errorf("%v", err)
		return
	}

	if err := tu.WriteVersionedBlobRA(bs, "backendonly", 10); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := bs.Sync(); err != nil {
		t.Errorf("Sync failed: %v", err)
		return
	}

	if err := tu.AssertBlobVersionRA(bs, "backendonly", 10); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.AssertBlobVersion(cachebs, "backendonly", 10); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.AssertBlobVersion(backendbs, "backendonly", 10); err != nil {
		t.Errorf("%v", err)
		return
	}
}
Exemplo n.º 4
0
func TestScheduler_RunTask(t *testing.T) {
	counter = 0

	s := scheduler.NewScheduler()
	s.RunImmediately(HogeTask{}, nil)
	s.RunAllAndStop()

	if counter != 1 {
		t.Errorf("err")
	}
}
Exemplo n.º 5
0
func TestScheduler_AbortAllAndStop(t *testing.T) {
	s := scheduler.NewScheduler()

	s.RunImmediately(LongTask(time.Second), nil)
	s.RunAt(LongTask(time.Second), time.Now().Add(300*time.Millisecond), nil)

	start := time.Now()
	s.AbortAllAndStop()
	if time.Since(start) > time.Second {
		t.Errorf("took >1sec to abort all and stop")
	}
}
Exemplo n.º 6
0
func TestScheduler_RunAt(t *testing.T) {
	counter = 0

	start := time.Now()
	s := scheduler.NewScheduler()

	cbjoin := make(chan struct{})
	cb := func(v *scheduler.JobView) {
		if v.State != scheduler.JobFinished {
			t.Errorf("wrong state")
		}
		if v.CreatedAt.Before(start) {
			t.Errorf("wrong CreatedAt")
		}
		if v.ScheduledAt.Before(v.CreatedAt) {
			t.Errorf("wrong ScheduledAt")
		}
		if v.Result.(HogeResult).counterSnapshot != 1 {
			t.Errorf("wrong result")
		}
		cbjoin <- struct{}{}
	}
	id := s.RunAt(HogeTask{}, time.Now().Add(100*time.Millisecond), cb)
	if id != scheduler.ID(1) {
		t.Errorf("first task not ID 1 but %d", id)
	}

	v := s.Query(id)
	if v == nil {
		t.Errorf("id should be queryable immediately after RunAt return")
		return
	}

	if v.ID != id {
		t.Errorf("ID mismatch")
	}
	if v.State != scheduler.JobScheduled {
		t.Errorf("wrong state")
	}
	if v.Result != nil {
		t.Errorf("result non-nil before run")
	}

	s.RunAllAndStop()

	if counter != 1 {
		t.Errorf("err")
	}
	<-cbjoin
}
Exemplo n.º 7
0
func TestScheduler_QueryAll(t *testing.T) {
	s := scheduler.NewScheduler()

	s.RunImmediately(LongTask(time.Second), nil)
	s.RunImmediately(LongTask(time.Second), nil)
	s.RunImmediately(LongTask(time.Second), nil)

	jvs := s.QueryAll()
	if len(jvs) != 3 {
		t.Errorf("Expected to see 3 jobs but saw %d", len(jvs))
	}

	s.AbortAllAndStop()
}
Exemplo n.º 8
0
func TestScheduler_AbortTaskAfterRun(t *testing.T) {
	s := scheduler.NewScheduler()

	cbjoin := make(chan struct{})
	cb := func(v *scheduler.JobView) {
		if v.State != scheduler.JobFinished {
			t.Errorf("wrong State")
		}
		if v.Result == nil {
			t.Errorf("Finished task has nil Result")
		}
		cbjoin <- struct{}{}
	}
	id := s.RunImmediately(HogeTask{}, cb)
	<-cbjoin
	s.Abort(id)

	s.RunAllAndStop()
}
Exemplo n.º 9
0
func TestScheduler_AbortTaskBeforeRun(t *testing.T) {
	s := scheduler.NewScheduler()

	cbjoin := make(chan struct{})
	cb := func(v *scheduler.JobView) {
		if v.State != scheduler.JobAborted {
			t.Errorf("wrong State")
		}
		if v.Result != nil {
			t.Errorf("Aborted task has non-nil Result")
		}
		cbjoin <- struct{}{}
	}
	id := s.RunAt(LongTask(time.Second), time.Now().Add(100*time.Millisecond), cb)
	s.Abort(id)

	s.RunAllAndStop()
	<-cbjoin
}
Exemplo n.º 10
0
func TestCachedBlobStore_ListBlobs(t *testing.T) {
	backendbs := tu.TestFileBlobStoreOfName("backend")
	cachebs := tu.TestFileBlobStoreOfName("cache")
	s := scheduler.NewScheduler()

	bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
	if err != nil {
		t.Errorf("Failed to create CachedBlobStore: %v", err)
		return
	}

	if err := tu.WriteVersionedBlob(backendbs, "backendonly", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.WriteVersionedBlob(cachebs, "cacheonly", 2); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.WriteVersionedBlobRA(bs, "synced", 3); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := bs.Sync(); err != nil {
		t.Errorf("Sync failed: %v", err)
		return
	}
	if err := tu.WriteVersionedBlobRA(bs, "unsynced", 4); err != nil {
		t.Errorf("%v", err)
		return
	}

	bpaths, err := bs.ListBlobs()
	if err != nil {
		t.Errorf("ListBlobs failed: %v", err)
		return
	}
	sort.Strings(bpaths)
	if !reflect.DeepEqual([]string{"backendonly", "synced", "unsynced"}, bpaths) {
		t.Errorf("ListBlobs returned unexpected result: %v", bpaths)
	}
}
Exemplo n.º 11
0
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()

	if !cfg.LocalDebug {
		o.Clisrc, err = auth.GetGCloudClientSource(
			path.Join(os.Getenv("HOME"), ".otaru", "credentials.json"),
			path.Join(os.Getenv("HOME"), ".otaru", "tokencache.json"),
			false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Clisrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Clisrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)

	o.SIO = otaru.NewBlobStoreDBStateSnapshotIO(o.CBS, o.C)

	if !cfg.LocalDebug {
		o.TxIO, err = datastore.NewDBTransactionLogIO(cfg.ProjectName, cfg.BucketName, o.C, o.Clisrc)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
		err = nil
	}
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init gcloud DBTransactionLogIO: %v", err)
	}

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.TxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.TxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
	o.MGMT = mgmt.NewServer()
	o.setupMgmtAPIs()
	if err := o.runMgmtServer(); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}
Exemplo n.º 12
0
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) {
	o := &Otaru{}

	var err error

	key := btncrypt.KeyFromPassword(cfg.Password)
	o.C, err = btncrypt.NewCipher(key)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init Cipher: %v", err)
	}

	o.S = scheduler.NewScheduler()

	if !cfg.LocalDebug {
		o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err)
		}
		o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc)
		o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info")
		if err := o.GL.Lock(); err != nil {
			return nil, err
		}
	}

	o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err)
	}

	if !cfg.LocalDebug {
		o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err)
		}
		if !cfg.UseSeparateBucketForMetadata {
			o.BackendBS = o.DefaultBS
		} else {
			metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName)
			o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE)
			if err != nil {
				o.Close()
				return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err)
			}

			o.BackendBS = blobstore.Mux{
				blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS},
				blobstore.MuxEntry{nil, o.DefaultBS},
			}
		}
	} else {
		o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE)
	}

	queryFn := chunkstore.NewQueryChunkVersion(o.C)
	o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn)
	if err != nil {
		o.Close()
		return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err)
	}
	if err := o.CBS.RestoreState(o.C); err != nil {
		logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err)
	}
	o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS)

	if !cfg.LocalDebug {
		o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg)
	} else {
		logger.Panicf(mylog, "Implement mock sslocator that doesn't depend on gcloud/datastore")
	}
	o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc)

	if !cfg.LocalDebug {
		txio := datastore.NewDBTransactionLogIO(o.DSCfg)
		o.TxIO = txio
		o.TxIOSS = util.NewSyncScheduler(txio, 300*time.Millisecond)
	} else {
		o.TxIO = inodedb.NewSimpleDBTransactionLogIO()
	}
	o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO)

	if oneshotcfg.Mkfs {
		o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewEmptyDB failed: %v", err)
		}
	} else {
		o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO)
		if err != nil {
			o.Close()
			return nil, fmt.Errorf("NewDB failed: %v", err)
		}
	}

	o.IDBS = inodedb.NewDBService(o.IDBBE)
	o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second)

	o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C)
	o.MGMT = mgmt.NewServer()
	if err := o.runMgmtServer(); err != nil {
		o.Close()
		return nil, fmt.Errorf("Mgmt server run failed: %v", err)
	}

	return o, nil
}
Exemplo n.º 13
0
func TestCachedBlobStore_RemoveBlob(t *testing.T) {
	backendbs := tu.TestFileBlobStoreOfName("backend")
	cachebs := tu.TestFileBlobStoreOfName("cache")
	s := scheduler.NewScheduler()

	bs, err := cachedblobstore.New(backendbs, cachebs, s, flags.O_RDWRCREATE, tu.TestQueryVersion)
	if err != nil {
		t.Errorf("Failed to create CachedBlobStore: %v", err)
		return
	}

	if err := tu.WriteVersionedBlob(backendbs, "backendonly", 1); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.WriteVersionedBlob(cachebs, "cacheonly", 2); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := tu.WriteVersionedBlobRA(bs, "synced", 3); err != nil {
		t.Errorf("%v", err)
		return
	}
	if err := bs.Sync(); err != nil {
		t.Errorf("Sync failed: %v", err)
		return
	}
	if err := tu.WriteVersionedBlobRA(bs, "unsynced", 4); err != nil {
		t.Errorf("%v", err)
		return
	}

	if err := bs.RemoveBlob("backendonly"); err != nil {
		t.Errorf("RemoveBlob failed: %v", err)
		return
	}
	if err := bs.RemoveBlob("synced"); err != nil {
		t.Errorf("RemoveBlob failed: %v", err)
		return
	}
	if err := bs.RemoveBlob("unsynced"); err != nil {
		t.Errorf("RemoveBlob failed: %v", err)
		return
	}

	bpaths, err := bs.ListBlobs()
	if err != nil {
		t.Errorf("ListBlobs failed: %v", err)
		return
	}
	if len(bpaths) > 0 {
		t.Errorf("Left over blobs: %v", bpaths)
	}

	for _, bp := range []string{"backendonly", "synced", "unsynced"} {
		if err := tu.AssertBlobVersionRA(bs, bp, 0); err != nil {
			t.Errorf("left over blob in bs: %s", bp)
		}
		if err := tu.AssertBlobVersion(cachebs, bp, 0); err != nil {
			t.Errorf("left over blob in cachebs: %s", bp)
		}
		if err := tu.AssertBlobVersion(backendbs, bp, 0); err != nil {
			t.Errorf("left over blob in backendbs: %s", bp)
		}
	}
}
Exemplo n.º 14
0
func TestScheduler_RunAllAndStop(t *testing.T) {
	s := scheduler.NewScheduler()
	s.RunAllAndStop()
}