Beispiel #1
0
func (be *CachedBlobEntry) Sync() error {
	be.mu.Lock()
	defer be.mu.Unlock()

	// Wait for invalidation to complete
	for be.state == cacheEntryInvalidating {
		logger.Infof(mylog, "Waiting for cache to be fully invalidated before sync.")
		be.validlenExtended.Wait()
	}

	if !be.state.IsActive() {
		logger.Warningf(mylog, "Attempted to sync already uninitialized/closed entry: %+v", be.infoWithLock())
		return nil
	}
	if be.state == cacheEntryClean {
		return nil
	}

	logger.Infof(mylog, "Sync entry: %+v", be.infoWithLock())

	errC := make(chan error)

	go func() {
		if err := be.writeBackWithLock(); err != nil {
			errC <- fmt.Errorf("Failed to writeback dirty: %v", err)
		} else {
			errC <- nil
		}
	}()

	go func() {
		if cs, ok := be.cachebh.(util.Syncer); ok {
			if err := cs.Sync(); err != nil {
				errC <- fmt.Errorf("Failed to sync cache blob: %v", err)
			} else {
				errC <- nil
			}
		} else {
			errC <- nil
		}
	}()

	errs := []error{}
	for i := 0; i < 2; i++ {
		if err := <-errC; err != nil {
			errs = append(errs, err)
		}
	}

	be.syncCount++
	be.lastSync = time.Now()
	return util.ToErrors(errs)
}
Beispiel #2
0
func (fs *FileSystem) OpenFile(id inodedb.ID, flags int) (*FileHandle, error) {
	logger.Infof(fslog, "OpenFile(id: %v, flags rok: %t wok: %t)", id, fl.IsReadAllowed(flags), fl.IsWriteAllowed(flags))

	tryLock := fl.IsWriteAllowed(flags)
	if tryLock && !fl.IsWriteAllowed(fs.bs.Flags()) {
		return nil, EACCES
	}

	of := fs.getOrCreateOpenFile(id)

	of.mu.Lock()
	defer of.mu.Unlock()

	ofIsInitialized := of.nlock.ID != 0
	if ofIsInitialized && (of.nlock.HasTicket() || !tryLock) {
		// No need to upgrade lock. Just use cached filehandle.
		logger.Infof(fslog, "Using cached of for inode id: %v", id)
		return of.OpenHandleWithoutLock(flags), nil
	}

	// upgrade lock or acquire new lock...
	v, nlock, err := fs.idb.QueryNode(id, tryLock)
	if err != nil {
		return nil, err
	}
	if v.GetType() != inodedb.FileNodeT {
		if err := fs.idb.UnlockNode(nlock); err != nil {
			logger.Warningf(fslog, "Unlock node failed for non-file node: %v", err)
		}

		if v.GetType() == inodedb.DirNodeT {
			return nil, EISDIR
		}
		return nil, fmt.Errorf("Specified node not file but has type %v", v.GetType())
	}

	of.nlock = nlock
	caio := NewINodeDBChunksArrayIO(fs.idb, nlock)
	of.cfio = chunkstore.NewChunkedFileIO(fs.bs, fs.c, caio)
	of.cfio.SetOrigFilename(fs.tryGetOrigPath(nlock.ID))

	if fl.IsWriteTruncate(flags) {
		if err := of.truncateWithLock(0); err != nil {
			return nil, fmt.Errorf("Failed to truncate file: %v", err)
		}
	}

	fh := of.OpenHandleWithoutLock(flags)
	return fh, nil
}
Beispiel #3
0
func SetupFluentLogger(cfg *Config) error {
	if cfg.Fluent.FluentHost == "" {
		logger.Infof(mylog, "The fluentd host is not specified. Skipping fluent logger instantiation.")
		return nil
	}
	logger.Infof(mylog, "Initializing fluent logger based on config: %+v", cfg.Fluent)

	fcli, err := gfluent.New(cfg.Fluent)
	if err != nil {
		return err
	}

	logger.Registry().AddOutput(fluent.FluentLogger{fcli})
	return nil
}
Beispiel #4
0
func (f *FileBlobStore) TotalSize() (int64, error) {
	start := time.Now()

	d, err := os.Open(f.base)
	if err != nil {
		return 0, fmt.Errorf("Open dir failed: %v", err)
	}
	defer d.Close()
	fis, err := d.Readdir(-1)
	if err != nil {
		return 0, fmt.Errorf("Readdir failed: %v", err)
	}

	totalSize := int64(0)
	for _, fi := range fis {
		if !fi.Mode().IsRegular() {
			continue
		}

		totalSize += fi.Size()
	}

	logger.Infof(mylog, "FileBlobStore.TotalSize() was %s. took %s.", humanize.Bytes(uint64(totalSize)), time.Since(start))
	return totalSize, nil
}
Beispiel #5
0
func (be *CachedBlobEntry) PWrite(p []byte, offset int64) error {
	be.mu.Lock()
	defer be.mu.Unlock()

	// Avoid any write when in invalidating state.
	// FIXME: maybe allow when offset+len(p) < be.validlen
	for be.state == cacheEntryInvalidating {
		logger.Infof(mylog, "Waiting for cache to be fully invalidated before write.")
		be.validlenExtended.Wait()
	}

	if len(p) == 0 {
		return nil
	}
	be.markDirtyWithLock()
	if err := be.cachebh.PWrite(p, offset); err != nil {
		return err
	}

	right := offset + int64(len(p))
	if right > be.bloblen {
		be.bloblen = right
		be.validlen = right
	}
	return nil
}
Beispiel #6
0
func (loc *INodeDBSSLocator) tryPutOnce(blobpath string, txid int64) error {
	start := time.Now()
	e := sslocentry{BlobPath: blobpath, TxID: txid, CreatedAt: start}

	cli, err := loc.cfg.getClient(context.TODO())
	if err != nil {
		return err
	}
	dstx, err := cli.NewTransaction(context.TODO(), datastore.Serializable)
	if err != nil {
		return err
	}

	key := datastore.NewKey(ctxNoNamespace, kindINodeDBSS, "", int64(e.TxID), loc.rootKey)
	if _, err := dstx.Put(key, &e); err != nil {
		dstx.Rollback()
		return err
	}
	if _, err := dstx.Commit(); err != nil {
		return err
	}

	logger.Infof(sslog, "Put(%s, %d) took %s.", blobpath, txid, time.Since(start))
	return nil
}
Beispiel #7
0
func (be *CachedBlobEntry) closeWithLock(abandon bool) error {
	if len(be.handles) > 0 {
		return fmt.Errorf("Entry has %d handles", len(be.handles))
	}

	logger.Infof(mylog, "Close entry: %+v", be.infoWithLock())

	if !abandon {
		for be.state == cacheEntryInvalidating {
			logger.Warningf(mylog, "Waiting for cache to be fully invalidated before close. (shouldn't come here, as PWrite should block)")
			be.validlenExtended.Wait()
		}

		if err := be.writeBackWithLock(); err != nil {
			return fmt.Errorf("Failed to writeback dirty: %v", err)
		}
		be.syncCount++
		be.lastSync = time.Now()
	}

	if err := be.cachebh.Close(); err != nil {
		return fmt.Errorf("Failed to close cache bh: %v", err)
	}

	be.state = cacheEntryClosed
	return nil
}
Beispiel #8
0
func (loc *INodeDBSSLocator) tryLocateOnce(history int) (string, error) {
	start := time.Now()
	cli, err := loc.cfg.getClient(context.TODO())
	if err != nil {
		return "", err
	}
	dstx, err := cli.NewTransaction(context.TODO(), datastore.Serializable)
	if err != nil {
		return "", err
	}

	q := datastore.NewQuery(kindINodeDBSS).Ancestor(loc.rootKey).Order("-TxID").Offset(history).Limit(1).Transaction(dstx)
	it := cli.Run(context.TODO(), q)
	var e sslocentry
	if _, err := it.Next(&e); err != nil {
		dstx.Rollback()
		if err == datastore.Done {
			return "", EEMPTY
		}
		return "", err
	}

	if _, err := dstx.Commit(); err != nil {
		return "", err
	}

	logger.Infof(sslog, "LocateSnapshot(%d) took %s. Found entry: %+v", history, time.Since(start), e)
	return e.BlobPath, nil
}
Beispiel #9
0
func (db *DB) RestoreVersion(version TxID) error {
	logger.Infof(mylog, "RestoreVersion(%s) start.", version)

	state, err := db.snapshotIO.RestoreSnapshot()
	if err != nil {
		return fmt.Errorf("Failed to restore snapshot: %v", err)
	}

	oldState := db.state
	db.state = state

	ssver := state.version
	logger.Infof(mylog, "Restored snapshot of ver %d.", ssver)

	if state.version > version {
		return fmt.Errorf("Can't rollback to old version %d which is older than snapshot version %d", version, state.version)
	}
	logger.Infof(mylog, "RestoreVersion(%s): restored ver: %s", version, ssver)

	txlog, err := db.txLogIO.QueryTransactions(ssver + 1)
	if txlog == nil || err != nil {
		db.state = oldState
		return fmt.Errorf("Failed to query txlog: %v", err)
	}

	for _, tx := range txlog {
		logger.Debugf(mylog, "RestoreVersion(%s): apply tx ver %s", version, tx.TxID)
		if _, err := db.applyTransactionInternal(tx, skipTxLog); err != nil {
			db.state = oldState
			return fmt.Errorf("Failed to replay tx: %v", err)
		}
	}

	logger.Infof(mylog, "Fast forward txlog from ver %d to %d", ssver, state.version)

	return nil
}
Beispiel #10
0
func (be *CachedBlobEntry) PRead(p []byte, offset int64) error {
	// FIXME: may be we should allow stale reads w/o lock
	be.mu.Lock()
	defer be.mu.Unlock()

	be.lastUsed = time.Now()

	requiredlen := util.Int64Min(offset+int64(len(p)), be.bloblen)
	for be.validlen < requiredlen {
		logger.Infof(mylog, "Waiting for cache to be fulfilled: reqlen: %d, validlen: %d", requiredlen, be.validlen)
		be.validlenExtended.Wait()
	}

	return be.cachebh.PRead(p, offset)
}
Beispiel #11
0
func (of *OpenFile) downgradeToReadLock() {
	logger.Infof(fslog, "Downgrade %v to read lock.", of)
	// Note: assumes of.mu is Lock()-ed

	if !of.nlock.HasTicket() {
		logger.Warningf(fslog, "Attempt to downgrade node lock, but no excl lock found. of: %v", of)
		return
	}

	if err := of.fs.idb.UnlockNode(of.nlock); err != nil {
		logger.Warningf(fslog, "Unlocking node to downgrade to read lock failed: %v", err)
	}
	of.nlock.Ticket = inodedb.NoTicket
	caio := NewINodeDBChunksArrayIO(of.fs.idb, of.nlock)
	of.cfio = chunkstore.NewChunkedFileIO(of.fs.bs, of.fs.c, caio)
}
Beispiel #12
0
func (loc *INodeDBSSLocator) DeleteAll() ([]string, error) {
	start := time.Now()

	cli, err := loc.cfg.getClient(context.TODO())
	if err != nil {
		return nil, err
	}

	dstx, err := cli.NewTransaction(context.TODO(), datastore.Serializable)
	if err != nil {
		return nil, err
	}

	keys := make([]*datastore.Key, 0)
	blobpaths := make([]string, 0)
	q := datastore.NewQuery(kindINodeDBSS).Ancestor(loc.rootKey).Transaction(dstx)
	it := cli.Run(context.TODO(), q)
	for {
		var e sslocentry
		k, err := it.Next(&e)
		if err != nil {
			if err == datastore.Done {
				break
			}
			dstx.Rollback()
			return nil, err
		}

		keys = append(keys, k)
		blobpaths = append(blobpaths, e.BlobPath)
	}

	logger.Debugf(sslog, "keys to delete: %v", keys)
	if err := dstx.DeleteMulti(keys); err != nil {
		dstx.Rollback()
		return nil, err
	}

	if _, err := dstx.Commit(); err != nil {
		return nil, err
	}
	logger.Infof(sslog, "DeleteAll() deleted %d entries. Took %s", len(keys), time.Since(start))
	return blobpaths, nil
}
Beispiel #13
0
func ServeFUSE(bucketName string, mountpoint string, ofs *otaru.FileSystem, ready chan<- bool) error {
	fsName := fmt.Sprintf("otaru+gs://%s", bucketName)
	volName := fmt.Sprintf("Otaru %s", bucketName)

	c, err := bfuse.Mount(
		mountpoint,
		bfuse.FSName(fsName),
		bfuse.Subtype("otarufs"),
		bfuse.VolumeName(volName),
	)
	if err != nil {
		return fmt.Errorf("bfuse.Mount failed: %v", err)
	}
	defer c.Close()

	serveC := make(chan error)
	go func() {
		if err := bfs.Serve(c, FileSystem{ofs}); err != nil {
			serveC <- err
			close(serveC)
			return
		}
		close(serveC)
	}()

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		return err
	}

	logger.Infof(mylog, "Mountpoint \"%s\" should be ready now!", mountpoint)
	if ready != nil {
		close(ready)
	}

	if err := <-serveC; err != nil {
		return nil
	}
	if err := ofs.Sync(); err != nil {
		return fmt.Errorf("Failed to Sync fs: %v", err)
	}
	return nil
}
Beispiel #14
0
func (be *CachedBlobEntry) Truncate(newsize int64) error {
	be.mu.Lock()
	defer be.mu.Unlock()

	// Avoid truncate when in invalidating state.
	// FIXME: maybe allow if newsize < be.validlen
	for be.state == cacheEntryInvalidating {
		logger.Infof(mylog, "Waiting for cache to be fully invalidated before truncate.")
		be.validlenExtended.Wait()
	}

	if be.bloblen == newsize {
		return nil
	}
	be.markDirtyWithLock()
	if err := be.cachebh.Truncate(newsize); err != nil {
		return err
	}
	be.bloblen = newsize
	be.validlen = newsize
	return nil
}
Beispiel #15
0
func (fs *FileSystem) SetAttr(id inodedb.ID, a Attr, valid ValidAttrFields) error {
	logger.Infof(fslog, "SetAttr id: %d, a: %+v, valid: %s", id, a, valid)

	ops := make([]inodedb.DBOperation, 0, 4)
	if valid&UidValid != 0 {
		ops = append(ops, &inodedb.UpdateUidOp{ID: id, Uid: a.Uid})
	}
	if valid&GidValid != 0 {
		ops = append(ops, &inodedb.UpdateGidOp{ID: id, Gid: a.Gid})
	}
	if valid&PermModeValid != 0 {
		ops = append(ops, &inodedb.UpdatePermModeOp{ID: id, PermMode: a.PermMode})
	}
	if valid&ModifiedTValid != 0 {
		ops = append(ops, &inodedb.UpdateModifiedTOp{ID: id, ModifiedT: a.ModifiedT})
	}

	if _, err := fs.idb.ApplyTransaction(inodedb.DBTransaction{Ops: ops}); err != nil {
		return err
	}
	return nil
}
Beispiel #16
0
func (f *FileBlobStore) ListBlobs() ([]string, error) {
	start := time.Now()

	d, err := os.Open(f.base)
	if err != nil {
		return nil, fmt.Errorf("Open dir failed: %v", err)
	}
	defer d.Close()
	fis, err := d.Readdir(-1)
	if err != nil {
		return nil, fmt.Errorf("Readdir failed: %v", err)
	}

	blobs := make([]string, 0, len(fis))
	for _, fi := range fis {
		if !fi.Mode().IsRegular() {
			continue
		}
		blobs = append(blobs, fi.Name())
	}

	logger.Infof(mylog, "FileBlobStore.ListBlobs() found %d blobs, took %s.", len(blobs), time.Since(start))
	return blobs, nil
}
Beispiel #17
0
func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	logger.Registry().AddOutput(logger.WriterLogger{os.Stderr})
	flag.Usage = Usage
	flag.Parse()

	cfg, err := facade.NewConfig(*flagConfigDir)
	if err != nil {
		logger.Criticalf(mylog, "%v", err)
		Usage()
		os.Exit(2)
	}
	if flag.NArg() != 1 {
		Usage()
		os.Exit(2)
	}
	mountpoint := flag.Arg(0)

	if err := facade.SetupFluentLogger(cfg); err != nil {
		logger.Criticalf(mylog, "Failed to setup fluentd logger: %v", err)
		os.Exit(1)
	}

	o, err := facade.NewOtaru(cfg, &facade.OneshotConfig{Mkfs: *flagMkfs})
	if err != nil {
		logger.Criticalf(mylog, "NewOtaru failed: %v", err)
		os.Exit(1)
	}
	var muClose sync.Mutex
	closeOtaruAndExit := func(exitCode int) {
		muClose.Lock()
		defer muClose.Unlock()

		if err := bfuse.Unmount(mountpoint); err != nil {
			logger.Warningf(mylog, "umount err: %v", err)
		}
		if o != nil {
			if err := o.Close(); err != nil {
				logger.Warningf(mylog, "Otaru.Close() returned errs: %v", err)
			}
			o = nil
		}
		os.Exit(exitCode)
	}
	defer closeOtaruAndExit(0)

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, os.Interrupt)
	signal.Notify(sigC, syscall.SIGTERM)
	go func() {
		for s := range sigC {
			logger.Warningf(mylog, "Received signal: %v", s)
			closeOtaruAndExit(1)
		}
	}()
	logger.Registry().AddOutput(logger.HandleCritical(func() {
		logger.Warningf(mylog, "Starting shutdown due to critical event.")
		closeOtaruAndExit(1)
	}))

	bfuseLogger := logger.Registry().Category("bfuse")
	bfuse.Debug = func(msg interface{}) { logger.Debugf(bfuseLogger, "%v", msg) }
	if err := fuse.ServeFUSE(cfg.BucketName, mountpoint, o.FS, nil); err != nil {
		logger.Warningf(mylog, "ServeFUSE failed: %v", err)
		closeOtaruAndExit(1)
	}
	logger.Infof(mylog, "ServeFUSE end!")
}
Beispiel #18
0
func GC(ctx context.Context, bs GCableBlobStore, idb inodedb.DBFscker, dryrun bool) error {
	start := time.Now()

	logger.Infof(mylog, "GC start. Dryrun: %t. Listing blobs.", dryrun)
	allbs, err := bs.ListBlobs()
	if err != nil {
		return fmt.Errorf("ListBlobs failed: %v", err)
	}
	logger.Infof(mylog, "List blobs done. %d blobs found.", len(allbs))
	if err := ctx.Err(); err != nil {
		logger.Infof(mylog, "Detected cancel. Bailing out.")
		return err
	}
	logger.Infof(mylog, "Starting INodeDB fsck.")
	usedbs, errs := idb.Fsck()
	if len(errs) != 0 {
		return fmt.Errorf("Fsck returned err: %v", err)
	}
	logger.Infof(mylog, "Fsck done. %d used blobs found.", len(usedbs))
	if err := ctx.Err(); err != nil {
		logger.Infof(mylog, "Detected cancel. Bailing out.")
		return err
	}

	logger.Infof(mylog, "Converting used blob list to a hashset")
	usedbset := make(map[string]struct{})
	for _, b := range usedbs {
		usedbset[b] = struct{}{}
	}
	logger.Infof(mylog, "Convert used blob list to a hashset: Done.")

	if err := ctx.Err(); err != nil {
		logger.Infof(mylog, "Detected cancel. Bailing out.")
		return err
	}

	logger.Infof(mylog, "Listing unused blobpaths.")
	unusedbs := make([]string, 0, util.IntMax(len(allbs)-len(usedbs), 0))
	for _, b := range allbs {
		if _, ok := usedbset[b]; ok {
			continue
		}

		if metadata.IsMetadataBlobpath(b) {
			logger.Infof(mylog, "Marking metadata blobpath as used: %s", b)
			continue
		}

		unusedbs = append(unusedbs, b)
	}

	traceend := time.Now()
	logger.Infof(mylog, "GC Found %d unused blobpaths. (Trace took %v)", len(unusedbs), traceend.Sub(start))

	for _, b := range unusedbs {
		if err := ctx.Err(); err != nil {
			logger.Infof(mylog, "Detected cancel. Bailing out.")
			return err
		}

		if dryrun {
			logger.Infof(mylog, "Dryrun found unused blob: %s", b)
		} else {
			logger.Infof(mylog, "Removing unused blob: %s", b)
			if err := bs.RemoveBlob(b); err != nil {
				return fmt.Errorf("Removing unused blob \"%s\" failed: %v", b, err)
			}
		}
	}
	sweepend := time.Now()
	logger.Infof(mylog, "GC success. Dryrun: %t. (Sweep took %v. The whole GC took %v.)", dryrun, sweepend.Sub(traceend), sweepend.Sub(start))

	return err
}
Beispiel #19
0
func (txio *DBTransactionLogIO) Sync() error {
	start := time.Now()

	txio.muSync.Lock()
	defer txio.muSync.Unlock()

	txio.mu.Lock()
	if len(txio.committing) != 0 {
		panic("I should be the only one committing.")
	}
	txio.committing = txio.nextbatch
	batch := txio.committing
	txio.nextbatch = make([]inodedb.DBTransaction, 0)
	txio.mu.Unlock()
	rollback := func() {
		txio.mu.Lock()
		txio.nextbatch = append(txio.committing, txio.nextbatch...)
		txio.committing = []inodedb.DBTransaction{}
		txio.mu.Unlock()
	}

	if len(batch) == 0 {
		return nil
	}

	cli, err := txio.cfg.getClient(context.Background())
	if err != nil {
		rollback()
		return err
	}

	keys := make([]*datastore.Key, 0, len(batch))
	stxs := make([]*storedbtx, 0, len(batch))
	for _, tx := range batch {
		key, stx, err := txio.encode(tx)
		if err != nil {
			rollback()
			return err
		}

		keys = append(keys, key)
		stxs = append(stxs, stx)
	}

	dstx, err := cli.NewTransaction(context.Background(), datastore.Serializable)
	if err != nil {
		rollback()
		return err
	}

	if _, err := dstx.PutMulti(keys, stxs); err != nil {
		rollback()
		dstx.Rollback()
		return err
	}

	if _, err := dstx.Commit(); err != nil {
		rollback()
		return err
	}

	txio.mu.Lock()
	txio.committing = []inodedb.DBTransaction{}
	txio.mu.Unlock()

	logger.Infof(txlog, "Sync() took %s. Committed %d txs", time.Since(start), len(stxs))
	return nil
}
Beispiel #20
0
func (txio *DBTransactionLogIO) DeleteTransactions(smallerThanID inodedb.TxID) error {
	start := time.Now()

	txio.mu.Lock()
	batch := make([]inodedb.DBTransaction, 0, len(txio.nextbatch))
	for _, tx := range txio.nextbatch {
		if tx.TxID < smallerThanID {
			continue
		}
		batch = append(batch, tx)
	}
	txio.nextbatch = batch
	txio.mu.Unlock()

	cli, err := txio.cfg.getClient(context.Background())
	if err != nil {
		return err
	}

	ndel := 0
	for {
		needAnotherTx := false
		txStart := time.Now()
		dstx, err := cli.NewTransaction(context.Background(), datastore.Serializable)
		if err != nil {
			return err
		}

		keys := []*datastore.Key{}
		ltkey := txio.encodeKey(smallerThanID)
		q := datastore.NewQuery(kindTransaction).Ancestor(txio.rootKey).Filter("__key__ <", ltkey).KeysOnly().Transaction(dstx)
		it := cli.Run(context.Background(), q)
		for {
			k, err := it.Next(nil)
			if err != nil {
				if err == datastore.Done {
					break
				}
				dstx.Rollback()
				return err
			}

			keys = append(keys, k)
			if len(keys) == maxWriteEntriesPerTx {
				needAnotherTx = true
				break
			}
		}

		if err := dstx.DeleteMulti(keys); err != nil {
			dstx.Rollback()
			return err
		}

		if _, err := dstx.Commit(); err != nil {
			return err
		}
		ndel += len(keys)

		if needAnotherTx {
			logger.Infof(txlog, "DeleteTransactions(%v): A tx deleting %d entries took %s. Starting next tx to delete more.", smallerThanID, len(keys), time.Since(txStart))
		} else {
			break
		}
	}
	logger.Infof(txlog, "DeleteTransactions(%v) deleted %d entries. tx took %s", smallerThanID, ndel, time.Since(start))
	return nil
}
Beispiel #21
0
func (cbs *CachedBlobStore) ReduceCache(ctx context.Context, desiredSize int64, dryrun bool) error {
	start := time.Now()

	tsizer, ok := cbs.cachebs.(blobstore.TotalSizer)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support TotalSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	blobsizer, ok := cbs.cachebs.(blobstore.BlobSizer)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support BlobSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	blobremover, ok := cbs.cachebs.(blobstore.BlobRemover)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support RemoveBlob() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	totalSizeBefore, err := tsizer.TotalSize()
	if err != nil {
		return fmt.Errorf("Failed to query current total cache size: %v", err)
	}

	needsReduce := totalSizeBefore - desiredSize
	if needsReduce < 0 {
		logger.Infof(mylog, "ReduceCache: No need to reduce cache as its already under desired size! No-op.")
		return nil
	}
	logger.Infof(mylog, "ReduceCache: Current cache bs total size: %s. Desired size: %s. Needs to reduce %s.",
		humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(desiredSize)), humanize.IBytes(uint64(needsReduce)))

	bps := cbs.usagestats.FindLeastUsed()
	for _, bp := range bps {
		size, err := blobsizer.BlobSize(bp)
		if err != nil {
			if os.IsNotExist(err) {
				logger.Infof(mylog, "Attempted to drop blob cache \"%s\", but not found. Maybe it's already removed.", bp)
				continue
			}
			return fmt.Errorf("Failed to query size for cache blob \"%s\": %v", bp, err)
		}

		logger.Infof(mylog, "ReduceCache: Drop entry \"%s\" to release %s", bp, humanize.IBytes(uint64(size)))

		if !dryrun {
			if err := cbs.entriesmgr.DropCacheEntry(bp, blobremover); err != nil {
				return fmt.Errorf("Failed to remove cache blob \"%s\": %v", bp, err)
			}
		}

		needsReduce -= size
		if needsReduce < 0 {
			break
		}
	}

	totalSizeAfter, err := tsizer.TotalSize()
	if err != nil {
		return fmt.Errorf("Failed to query current total cache size: %v", err)
	}

	logger.Infof(mylog, "ReduceCache done. Cache bs total size: %s -> %s. Dryrun: %t. Took: %s",
		humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(totalSizeAfter)),
		dryrun, time.Since(start))
	return nil
}
Beispiel #22
0
func (txio *DBTransactionLogIO) queryTransactionsOnce(minID inodedb.TxID) ([]inodedb.DBTransaction, error) {
	start := time.Now()
	txs := []inodedb.DBTransaction{}

	txio.mu.Lock()
	for _, tx := range txio.committing {
		if tx.TxID >= minID {
			txs = append(txs, tx)
		}
	}
	for _, tx := range txio.nextbatch {
		if tx.TxID >= minID {
			txs = append(txs, tx)
		}
	}
	txio.mu.Unlock()

	cli, err := txio.cfg.getClient(context.Background())
	if err != nil {
		return nil, err
	}

	dstx, err := cli.NewTransaction(context.Background(), datastore.Serializable)
	if err != nil {
		return nil, err
	}

	q := datastore.NewQuery(kindTransaction).Ancestor(txio.rootKey).Transaction(dstx)
	if minID != inodedb.AnyVersion {
		minKey := txio.encodeKey(minID)
		q = q.Filter("__key__ >=", minKey)
	}

	it := cli.Run(context.Background(), q)
	for {
		var stx storedbtx
		key, err := it.Next(&stx)
		if err != nil {
			if err == datastore.Done {
				break
			}
			dstx.Commit()
			return []inodedb.DBTransaction{}, err
		}

		tx, err := decode(txio.cfg.c, key, &stx)
		if err != nil {
			dstx.Commit()
			return []inodedb.DBTransaction{}, err
		}

		txs = append(txs, tx)
	}

	// FIXME: not sure if Rollback() is better
	if _, err := dstx.Commit(); err != nil {
		return nil, err
	}

	sort.Sort(txsorter(txs))
	uniqed := make([]inodedb.DBTransaction, 0, len(txs))
	var prevId inodedb.TxID
	for _, tx := range txs {
		if tx.TxID == prevId {
			continue
		}

		uniqed = append(uniqed, tx)
		prevId = tx.TxID
	}

	logger.Infof(txlog, "QueryTransactions(%v) took %s", minID, time.Since(start))
	return uniqed, nil
}