Пример #1
0
// Create a directory inode for the name, representing the directory containing
// the objects for which it is an immediate prefix. For the root directory,
// this is the empty string.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// If typeCacheTTL is non-zero, a cache from child name to information about
// whether that name exists as a file/symlink and/or directory will be
// maintained. This may speed up calls to LookUpChild, especially when combined
// with a stat-caching GCS bucket, but comes at the cost of consistency: if the
// child is removed and recreated with a different type before the expiration,
// we may fail to find it.
//
// The initial lookup count is zero.
//
// REQUIRES: IsDirName(name)
func NewDirInode(
	id fuseops.InodeID,
	name string,
	attrs fuseops.InodeAttributes,
	implicitDirs bool,
	typeCacheTTL time.Duration,
	bucket gcs.Bucket,
	clock timeutil.Clock) (d DirInode) {
	if !IsDirName(name) {
		panic(fmt.Sprintf("Unexpected name: %s", name))
	}

	// Set up the struct.
	const typeCacheCapacity = 1 << 16
	typed := &dirInode{
		bucket:       bucket,
		clock:        clock,
		id:           id,
		implicitDirs: implicitDirs,
		name:         name,
		attrs:        attrs,
		cache:        newTypeCache(typeCacheCapacity/2, typeCacheTTL),
	}

	typed.lc.Init(id)

	// Set up invariant checking.
	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	d = typed
	return
}
Пример #2
0
// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: o.MetaGeneration > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	id fuseops.InodeID,
	o *gcs.Object,
	attrs fuseops.InodeAttributes,
	bucket gcs.Bucket,
	syncer gcsx.Syncer,
	tempDir string,
	mtimeClock timeutil.Clock) (f *FileInode) {
	// Set up the basic struct.
	f = &FileInode{
		bucket:     bucket,
		syncer:     syncer,
		mtimeClock: mtimeClock,
		id:         id,
		name:       o.Name,
		attrs:      attrs,
		tempDir:    tempDir,
		src:        *o,
	}

	f.lc.Init(id)

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}
Пример #3
0
// Create a cache that holds the given number of items, evicting the least
// recently used item when more space is needed.
func NewLruCache(capacity uint) Cache {
	c := &lruCache{
		wrapped: lrucache.New(int(capacity)),
	}

	c.mu = syncutil.NewInvariantMutex(c.wrapped.CheckInvariants)
	return c
}
Пример #4
0
// Create an "in-memory GCS" that allows access to buckets of any name, each
// initially with empty contents. The supplied clock will be used for
// generating timestamps.
func NewConn(clock timeutil.Clock) (c gcs.Conn) {
	typed := &conn{
		clock:   clock,
		buckets: make(map[string]gcs.Bucket),
	}

	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	c = typed
	return
}
Пример #5
0
func newFileHandle(
	scores []blob.Score,
	blobStore blob.Store) (fh *fileHandle) {
	fh = &fileHandle{
		blobStore: blobStore,
		scores:    scores,
	}

	fh.mu = syncutil.NewInvariantMutex(fh.checkInvariants)

	return
}
Пример #6
0
func (c *lruCache) GobDecode(b []byte) (err error) {
	// Decode the wrapped cache.
	err = c.wrapped.GobDecode(b)
	if err != nil {
		return
	}

	// Initialize the mutex.
	c.mu = syncutil.NewInvariantMutex(c.wrapped.CheckInvariants)

	return
}
Пример #7
0
func NewFileHandle(
	inode *inode.FileInode,
	bucket gcs.Bucket) (fh *FileHandle) {
	fh = &FileHandle{
		inode:  inode,
		bucket: bucket,
	}

	fh.mu = syncutil.NewInvariantMutex(fh.checkInvariants)

	return
}
Пример #8
0
// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(
	in inode.DirInode,
	implicitDirs bool) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in:           in,
		implicitDirs: implicitDirs,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}
Пример #9
0
// Create an inode with the supplied attributes. The supplied scores should
// contain the inode's contents.
func newFileInode(
	attrs fuseops.InodeAttributes,
	scores []blob.Score,
	blobStore blob.Store) (f *fileInode) {
	f = &fileInode{
		scores:    scores,
		blobStore: blobStore,
		attrs:     attrs,
	}

	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}
Пример #10
0
// size is the size that the leaser has already recorded for us. It must match
// the file's size.
func newReadWriteLease(
	leaser *fileLeaser,
	size int64,
	file *os.File) (rwl *readWriteLease) {
	rwl = &readWriteLease{
		leaser:       leaser,
		file:         file,
		reportedSize: size,
		fileSize:     size,
	}

	rwl.mu = syncutil.NewInvariantMutex(rwl.checkInvariants)

	return
}
Пример #11
0
// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in number of files
// and bytes below the given limits. If dir is empty, the system default will be
// used.
//
// Usage may exceed the given limits if there are read/write leases whose total
// size exceeds the limits, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitNumFiles int,
	limitBytes int64) (fl FileLeaser) {
	typed := &fileLeaser{
		dir:             dir,
		limitNumFiles:   limitNumFiles,
		limitBytes:      limitBytes,
		readLeasesIndex: make(map[*readLease]*list.Element),
	}

	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	fl = typed
	return
}
Пример #12
0
// Create a file system whose sole contents are a file named "foo" and a
// directory named "bar".
//
// The file "foo" may be opened for reading and/or writing, but reads and
// writes aren't supported. Additionally, any non-existent file or directory
// name may be created within any directory, but the resulting inode will
// appear to have been unlinked immediately.
//
// The file system maintains reference counts for the inodes involved. It will
// panic if a reference count becomes negative or if an inode ID is re-used
// after we expect it to be dead. Its Check method may be used to check that
// there are no inodes with unexpected reference counts remaining, after
// unmounting.
func NewFileSystem() (fs *ForgetFS) {
	// Set up the actual file system.
	impl := &fsImpl{
		inodes: map[fuseops.InodeID]*inode{
			cannedID_Root: &inode{
				attributes: fuseops.InodeAttributes{
					Nlink: 1,
					Mode:  0777 | os.ModeDir,
				},
			},
			cannedID_Foo: &inode{
				attributes: fuseops.InodeAttributes{
					Nlink: 1,
					Mode:  0777,
				},
			},
			cannedID_Bar: &inode{
				attributes: fuseops.InodeAttributes{
					Nlink: 1,
					Mode:  0777 | os.ModeDir,
				},
			},
		},
		nextInodeID: cannedID_Next,
	}

	// The root inode starts with a lookup count of one.
	impl.inodes[cannedID_Root].IncrementLookupCount()

	// The canned inodes are supposed to be stable from the user's point of view,
	// so we should allow them to be looked up at any point even if the kernel
	// has balanced its lookups with its forgets. Ensure that they never go to
	// zero until the file system is destroyed.
	impl.inodes[cannedID_Foo].IncrementLookupCount()
	impl.inodes[cannedID_Bar].IncrementLookupCount()

	// Set up the mutex.
	impl.mu = syncutil.NewInvariantMutex(impl.checkInvariants)

	// Set up a wrapper that exposes only certain methods.
	fs = &ForgetFS{
		impl:   impl,
		server: fuseutil.NewFileSystemServer(impl),
	}

	return
}
Пример #13
0
// Create a file system that issues cacheable responses according to the
// following rules:
//
//  *  LookUpInodeResponse.Entry.EntryExpiration is set according to
//     lookupEntryTimeout.
//
//  *  GetInodeAttributesResponse.AttributesExpiration is set according to
//     getattrTimeout.
//
//  *  Nothing else is marked cacheable. (In particular, the attributes
//     returned by LookUpInode are not cacheable.)
//
func NewCachingFS(
	lookupEntryTimeout time.Duration,
	getattrTimeout time.Duration) (fs CachingFS, err error) {
	roundUp := func(n fuseops.InodeID) fuseops.InodeID {
		return numInodes * ((n + numInodes - 1) / numInodes)
	}

	cfs := &cachingFS{
		lookupEntryTimeout: lookupEntryTimeout,
		getattrTimeout:     getattrTimeout,
		baseID:             roundUp(fuseops.RootInodeID + 1),
		mtime:              time.Now(),
	}

	cfs.mu = syncutil.NewInvariantMutex(cfs.checkInvariants)

	fs = cfs
	return
}
Пример #14
0
// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// gcsChunkSize controls the maximum size of each individual read request made
// to GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
	id fuseops.InodeID,
	o *gcs.Object,
	attrs fuseops.InodeAttributes,
	gcsChunkSize uint64,
	bucket gcs.Bucket,
	leaser lease.FileLeaser,
	objectSyncer gcsproxy.ObjectSyncer,
	clock timeutil.Clock) (f *FileInode) {
	// Set up the basic struct.
	f = &FileInode{
		bucket:       bucket,
		leaser:       leaser,
		objectSyncer: objectSyncer,
		clock:        clock,
		id:           id,
		name:         o.Name,
		attrs:        attrs,
		gcsChunkSize: gcsChunkSize,
		src:          *o,
		content: mutable.NewContent(
			gcsproxy.NewReadProxy(
				o,
				nil, // Initial read lease
				gcsChunkSize,
				leaser,
				bucket),
			clock),
	}

	f.lc.Init(id)

	// Set up invariant checking.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}
Пример #15
0
// Create a file system that stores data and metadata in memory.
//
// The supplied UID/GID pair will own the root inode. This file system does no
// permissions checking, and should therefore be mounted with the
// default_permissions option.
func NewMemFS(
	uid uint32,
	gid uint32) fuse.Server {
	// Set up the basic struct.
	fs := &memFS{
		inodes: make([]*inode, fuseops.RootInodeID+1),
		uid:    uid,
		gid:    gid,
	}

	// Set up the root inode.
	rootAttrs := fuseops.InodeAttributes{
		Mode: 0700 | os.ModeDir,
		Uid:  uid,
		Gid:  gid,
	}

	fs.inodes[fuseops.RootInodeID] = newInode(rootAttrs)

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	return fuseutil.NewFileSystemServer(fs)
}
Пример #16
0
// Equivalent to NewConn(clock).GetBucket(name).
func NewFakeBucket(clock timeutil.Clock, name string) gcs.Bucket {
	b := &bucket{clock: clock, name: name}
	b.mu = syncutil.NewInvariantMutex(b.checkInvariants)
	return b
}
Пример #17
0
// Create a fuse file system server according to the supplied configuration.
func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
	// Check permissions bits.
	if cfg.FilePerms&^os.ModePerm != 0 {
		err = fmt.Errorf("Illegal file perms: %v", cfg.FilePerms)
		return
	}

	if cfg.DirPerms&^os.ModePerm != 0 {
		err = fmt.Errorf("Illegal dir perms: %v", cfg.FilePerms)
		return
	}

	// Create the object syncer.
	if cfg.TmpObjectPrefix == "" {
		err = errors.New("You must set TmpObjectPrefix.")
		return
	}

	syncer := gcsx.NewSyncer(
		cfg.AppendThreshold,
		cfg.TmpObjectPrefix,
		cfg.Bucket)

	// Set up the basic struct.
	fs := &fileSystem{
		mtimeClock:             timeutil.RealClock(),
		cacheClock:             cfg.CacheClock,
		bucket:                 cfg.Bucket,
		syncer:                 syncer,
		tempDir:                cfg.TempDir,
		implicitDirs:           cfg.ImplicitDirectories,
		inodeAttributeCacheTTL: cfg.InodeAttributeCacheTTL,
		dirTypeCacheTTL:        cfg.DirTypeCacheTTL,
		uid:                    cfg.Uid,
		gid:                    cfg.Gid,
		fileMode:               cfg.FilePerms,
		dirMode:                cfg.DirPerms | os.ModeDir,
		inodes:                 make(map[fuseops.InodeID]inode.Inode),
		nextInodeID:            fuseops.RootInodeID + 1,
		generationBackedInodes: make(map[string]inode.GenerationBackedInode),
		implicitDirInodes:      make(map[string]inode.DirInode),
		handles:                make(map[fuseops.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(
		fuseops.RootInodeID,
		"", // name
		fuseops.InodeAttributes{
			Uid:  fs.uid,
			Gid:  fs.gid,
			Mode: fs.dirMode,
		},
		fs.implicitDirs,
		fs.dirTypeCacheTTL,
		cfg.Bucket,
		fs.mtimeClock,
		fs.cacheClock)

	root.Lock()
	root.IncrementLookupCount()
	fs.inodes[fuseops.RootInodeID] = root
	fs.implicitDirInodes[root.Name()] = root
	root.Unlock()

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	// Periodically garbage collect temporary objects.
	var gcCtx context.Context
	gcCtx, fs.stopGarbageCollecting = context.WithCancel(context.Background())
	go garbageCollect(gcCtx, cfg.TmpObjectPrefix, fs.bucket)

	server = fuseutil.NewFileSystemServer(fs)
	return
}