// Create a directory inode for the name, representing the directory containing
// the objects for which it is an immediate prefix. For the root directory,
// this is the empty string.
//
// If implicitDirs is set, LookUpChild will use ListObjects to find child
// directories that are "implicitly" defined by the existence of their own
// descendents. For example, if there is an object named "foo/bar/baz" and this
// is the directory "foo", a child directory named "bar" will be implied.
//
// If typeCacheTTL is non-zero, a cache from child name to information about
// whether that name exists as a file/symlink and/or directory will be
// maintained. This may speed up calls to LookUpChild, especially when combined
// with a stat-caching GCS bucket, but comes at the cost of consistency: if the
// child is removed and recreated with a different type before the expiration,
// we may fail to find it.
//
// The initial lookup count is zero.
//
// REQUIRES: IsDirName(name)
func NewDirInode(
id fuseops.InodeID,
name string,
attrs fuseops.InodeAttributes,
implicitDirs bool,
typeCacheTTL time.Duration,
bucket gcs.Bucket,
clock timeutil.Clock) (d DirInode) {
if !IsDirName(name) {
panic(fmt.Sprintf("Unexpected name: %s", name))
}
// Set up the struct.
const typeCacheCapacity = 1 << 16
typed := &dirInode{
bucket: bucket,
clock: clock,
id: id,
implicitDirs: implicitDirs,
name: name,
attrs: attrs,
cache: newTypeCache(typeCacheCapacity/2, typeCacheTTL),
}
typed.lc.Init(id)
// Set up invariant checking.
typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)
d = typed
return
}
// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: o.MetaGeneration > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
id fuseops.InodeID,
o *gcs.Object,
attrs fuseops.InodeAttributes,
bucket gcs.Bucket,
syncer gcsx.Syncer,
tempDir string,
mtimeClock timeutil.Clock) (f *FileInode) {
// Set up the basic struct.
f = &FileInode{
bucket: bucket,
syncer: syncer,
mtimeClock: mtimeClock,
id: id,
name: o.Name,
attrs: attrs,
tempDir: tempDir,
src: *o,
}
f.lc.Init(id)
// Set up invariant checking.
f.mu = syncutil.NewInvariantMutex(f.checkInvariants)
return
}
// Create a cache that holds the given number of items, evicting the least
// recently used item when more space is needed.
func NewLruCache(capacity uint) Cache {
c := &lruCache{
wrapped: lrucache.New(int(capacity)),
}
c.mu = syncutil.NewInvariantMutex(c.wrapped.CheckInvariants)
return c
}
// Create an "in-memory GCS" that allows access to buckets of any name, each
// initially with empty contents. The supplied clock will be used for
// generating timestamps.
func NewConn(clock timeutil.Clock) (c gcs.Conn) {
typed := &conn{
clock: clock,
buckets: make(map[string]gcs.Bucket),
}
typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)
c = typed
return
}
func newFileHandle(
scores []blob.Score,
blobStore blob.Store) (fh *fileHandle) {
fh = &fileHandle{
blobStore: blobStore,
scores: scores,
}
fh.mu = syncutil.NewInvariantMutex(fh.checkInvariants)
return
}
func (c *lruCache) GobDecode(b []byte) (err error) {
// Decode the wrapped cache.
err = c.wrapped.GobDecode(b)
if err != nil {
return
}
// Initialize the mutex.
c.mu = syncutil.NewInvariantMutex(c.wrapped.CheckInvariants)
return
}
func NewFileHandle(
inode *inode.FileInode,
bucket gcs.Bucket) (fh *FileHandle) {
fh = &FileHandle{
inode: inode,
bucket: bucket,
}
fh.mu = syncutil.NewInvariantMutex(fh.checkInvariants)
return
}
// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(
in inode.DirInode,
implicitDirs bool) (dh *dirHandle) {
// Set up the basic struct.
dh = &dirHandle{
in: in,
implicitDirs: implicitDirs,
}
// Set up invariant checking.
dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)
return
}
// Create an inode with the supplied attributes. The supplied scores should
// contain the inode's contents.
func newFileInode(
attrs fuseops.InodeAttributes,
scores []blob.Score,
blobStore blob.Store) (f *fileInode) {
f = &fileInode{
scores: scores,
blobStore: blobStore,
attrs: attrs,
}
f.mu = syncutil.NewInvariantMutex(f.checkInvariants)
return
}
// size is the size that the leaser has already recorded for us. It must match
// the file's size.
func newReadWriteLease(
leaser *fileLeaser,
size int64,
file *os.File) (rwl *readWriteLease) {
rwl = &readWriteLease{
leaser: leaser,
file: file,
reportedSize: size,
fileSize: size,
}
rwl.mu = syncutil.NewInvariantMutex(rwl.checkInvariants)
return
}
// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in number of files
// and bytes below the given limits. If dir is empty, the system default will be
// used.
//
// Usage may exceed the given limits if there are read/write leases whose total
// size exceeds the limits, since such leases cannot be revoked.
func NewFileLeaser(
dir string,
limitNumFiles int,
limitBytes int64) (fl FileLeaser) {
typed := &fileLeaser{
dir: dir,
limitNumFiles: limitNumFiles,
limitBytes: limitBytes,
readLeasesIndex: make(map[*readLease]*list.Element),
}
typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)
fl = typed
return
}
// Create a file system whose sole contents are a file named "foo" and a
// directory named "bar".
//
// The file "foo" may be opened for reading and/or writing, but reads and
// writes aren't supported. Additionally, any non-existent file or directory
// name may be created within any directory, but the resulting inode will
// appear to have been unlinked immediately.
//
// The file system maintains reference counts for the inodes involved. It will
// panic if a reference count becomes negative or if an inode ID is re-used
// after we expect it to be dead. Its Check method may be used to check that
// there are no inodes with unexpected reference counts remaining, after
// unmounting.
func NewFileSystem() (fs *ForgetFS) {
// Set up the actual file system.
impl := &fsImpl{
inodes: map[fuseops.InodeID]*inode{
cannedID_Root: &inode{
attributes: fuseops.InodeAttributes{
Nlink: 1,
Mode: 0777 | os.ModeDir,
},
},
cannedID_Foo: &inode{
attributes: fuseops.InodeAttributes{
Nlink: 1,
Mode: 0777,
},
},
cannedID_Bar: &inode{
attributes: fuseops.InodeAttributes{
Nlink: 1,
Mode: 0777 | os.ModeDir,
},
},
},
nextInodeID: cannedID_Next,
}
// The root inode starts with a lookup count of one.
impl.inodes[cannedID_Root].IncrementLookupCount()
// The canned inodes are supposed to be stable from the user's point of view,
// so we should allow them to be looked up at any point even if the kernel
// has balanced its lookups with its forgets. Ensure that they never go to
// zero until the file system is destroyed.
impl.inodes[cannedID_Foo].IncrementLookupCount()
impl.inodes[cannedID_Bar].IncrementLookupCount()
// Set up the mutex.
impl.mu = syncutil.NewInvariantMutex(impl.checkInvariants)
// Set up a wrapper that exposes only certain methods.
fs = &ForgetFS{
impl: impl,
server: fuseutil.NewFileSystemServer(impl),
}
return
}
// Create a file system that issues cacheable responses according to the
// following rules:
//
// * LookUpInodeResponse.Entry.EntryExpiration is set according to
// lookupEntryTimeout.
//
// * GetInodeAttributesResponse.AttributesExpiration is set according to
// getattrTimeout.
//
// * Nothing else is marked cacheable. (In particular, the attributes
// returned by LookUpInode are not cacheable.)
//
func NewCachingFS(
lookupEntryTimeout time.Duration,
getattrTimeout time.Duration) (fs CachingFS, err error) {
roundUp := func(n fuseops.InodeID) fuseops.InodeID {
return numInodes * ((n + numInodes - 1) / numInodes)
}
cfs := &cachingFS{
lookupEntryTimeout: lookupEntryTimeout,
getattrTimeout: getattrTimeout,
baseID: roundUp(fuseops.RootInodeID + 1),
mtime: time.Now(),
}
cfs.mu = syncutil.NewInvariantMutex(cfs.checkInvariants)
fs = cfs
return
}
// Create a file inode for the given object in GCS. The initial lookup count is
// zero.
//
// gcsChunkSize controls the maximum size of each individual read request made
// to GCS.
//
// REQUIRES: o != nil
// REQUIRES: o.Generation > 0
// REQUIRES: len(o.Name) > 0
// REQUIRES: o.Name[len(o.Name)-1] != '/'
func NewFileInode(
id fuseops.InodeID,
o *gcs.Object,
attrs fuseops.InodeAttributes,
gcsChunkSize uint64,
bucket gcs.Bucket,
leaser lease.FileLeaser,
objectSyncer gcsproxy.ObjectSyncer,
clock timeutil.Clock) (f *FileInode) {
// Set up the basic struct.
f = &FileInode{
bucket: bucket,
leaser: leaser,
objectSyncer: objectSyncer,
clock: clock,
id: id,
name: o.Name,
attrs: attrs,
gcsChunkSize: gcsChunkSize,
src: *o,
content: mutable.NewContent(
gcsproxy.NewReadProxy(
o,
nil, // Initial read lease
gcsChunkSize,
leaser,
bucket),
clock),
}
f.lc.Init(id)
// Set up invariant checking.
f.mu = syncutil.NewInvariantMutex(f.checkInvariants)
return
}
// Create a file system that stores data and metadata in memory.
//
// The supplied UID/GID pair will own the root inode. This file system does no
// permissions checking, and should therefore be mounted with the
// default_permissions option.
func NewMemFS(
uid uint32,
gid uint32) fuse.Server {
// Set up the basic struct.
fs := &memFS{
inodes: make([]*inode, fuseops.RootInodeID+1),
uid: uid,
gid: gid,
}
// Set up the root inode.
rootAttrs := fuseops.InodeAttributes{
Mode: 0700 | os.ModeDir,
Uid: uid,
Gid: gid,
}
fs.inodes[fuseops.RootInodeID] = newInode(rootAttrs)
// Set up invariant checking.
fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)
return fuseutil.NewFileSystemServer(fs)
}
// Equivalent to NewConn(clock).GetBucket(name).
func NewFakeBucket(clock timeutil.Clock, name string) gcs.Bucket {
b := &bucket{clock: clock, name: name}
b.mu = syncutil.NewInvariantMutex(b.checkInvariants)
return b
}
// Create a fuse file system server according to the supplied configuration.
func NewServer(cfg *ServerConfig) (server fuse.Server, err error) {
// Check permissions bits.
if cfg.FilePerms&^os.ModePerm != 0 {
err = fmt.Errorf("Illegal file perms: %v", cfg.FilePerms)
return
}
if cfg.DirPerms&^os.ModePerm != 0 {
err = fmt.Errorf("Illegal dir perms: %v", cfg.FilePerms)
return
}
// Create the object syncer.
if cfg.TmpObjectPrefix == "" {
err = errors.New("You must set TmpObjectPrefix.")
return
}
syncer := gcsx.NewSyncer(
cfg.AppendThreshold,
cfg.TmpObjectPrefix,
cfg.Bucket)
// Set up the basic struct.
fs := &fileSystem{
mtimeClock: timeutil.RealClock(),
cacheClock: cfg.CacheClock,
bucket: cfg.Bucket,
syncer: syncer,
tempDir: cfg.TempDir,
implicitDirs: cfg.ImplicitDirectories,
inodeAttributeCacheTTL: cfg.InodeAttributeCacheTTL,
dirTypeCacheTTL: cfg.DirTypeCacheTTL,
uid: cfg.Uid,
gid: cfg.Gid,
fileMode: cfg.FilePerms,
dirMode: cfg.DirPerms | os.ModeDir,
inodes: make(map[fuseops.InodeID]inode.Inode),
nextInodeID: fuseops.RootInodeID + 1,
generationBackedInodes: make(map[string]inode.GenerationBackedInode),
implicitDirInodes: make(map[string]inode.DirInode),
handles: make(map[fuseops.HandleID]interface{}),
}
// Set up the root inode.
root := inode.NewDirInode(
fuseops.RootInodeID,
"", // name
fuseops.InodeAttributes{
Uid: fs.uid,
Gid: fs.gid,
Mode: fs.dirMode,
},
fs.implicitDirs,
fs.dirTypeCacheTTL,
cfg.Bucket,
fs.mtimeClock,
fs.cacheClock)
root.Lock()
root.IncrementLookupCount()
fs.inodes[fuseops.RootInodeID] = root
fs.implicitDirInodes[root.Name()] = root
root.Unlock()
// Set up invariant checking.
fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)
// Periodically garbage collect temporary objects.
var gcCtx context.Context
gcCtx, fs.stopGarbageCollecting = context.WithCancel(context.Background())
go garbageCollect(gcCtx, cfg.TmpObjectPrefix, fs.bucket)
server = fuseutil.NewFileSystemServer(fs)
return
}