// Lazy file loading
func (dir *Directory) loadChildren() {
util.P_out("Loading children")
for key := range dir.ChildVids {
util.P_out(key)
if dir.IsDir[key] {
child, err := filesystem.GetDirectory(dir.ChildVids[key], dir.Source)
if err != nil {
util.P_err("Error loading directory from db: ", err)
} else {
child.parent = dir
child.archive = dir.isArchive()
dir.setChild(child)
}
} else {
child, err := filesystem.GetFile(dir.ChildVids[key], dir.Source)
if err != nil {
util.P_err("Error loading file from db: ", err)
} else {
child.parent = dir
child.archive = dir.isArchive()
dir.setChild(child)
}
}
}
dir.childrenInMemory = true
}
func (node *Node) InitNode(name string, mode os.FileMode, parent *Directory) {
node.Source = filesystem.replInfo.Pid
node.Vid = NULL_VERSION
node.LastVid = NULL_VERSION
util.P_out("VID: ", node.Vid)
node.Attrs.Inode = filesystem.getNextInd()
node.Attrs.Nlink = 1
node.Name = name
tm := time.Now()
node.Attrs.Atime = tm
node.Attrs.Mtime = tm
node.Attrs.Ctime = tm
node.Attrs.Crtime = tm
node.Attrs.Mode = mode
node.Attrs.Gid = uint32(gid)
node.Attrs.Uid = uint32(uid)
node.Attrs.Size = 0
node.dirty = true
node.parent = parent
node.archive = false
util.P_out("inited node inode %d, %q\n", filesystem.NextInd, name)
}
func flushNode(flushTime time.Time, node NamedNode, changes map[string][]byte) {
if dir, ok := node.(*Directory); ok {
util.P_out("Visiting node: "+dir.Name+" at version %d", dir.Vid)
for _, node := range dir.children {
if node != nil {
if !node.isArchive() {
flushNode(flushTime, node, changes)
}
}
}
// TODO: make this a dir method or another function here
if dir.dirty {
util.P_out("Putting " + dir.Name)
dir.LastVid = dir.Vid
//dir.Vid = filesystem.getNextVid()
dir.Vid = dir.ComputeVid()
dir.Vtime = flushTime
dir.Source = filesystem.replInfo.Pid
if dir.parent != nil {
dir.parent.setChild(dir)
if err := filesystem.database.PutDirectory(dir); err != nil {
util.P_err("Error putting directory in db: ", err)
}
dir.parent.dirty = true
} else {
if err := filesystem.database.SetRoot(dir); err != nil {
util.P_err("Error setting root in db: ", err)
}
changes[HEAD] = dir.Vid
}
changes[hex.EncodeToString(dir.Vid)], _ = json.Marshal(dir)
}
}
if file, ok := node.(*File); ok {
util.P_out("Visiting " + file.Name)
}
// TODO: make this a file method or another function here
if file, ok := node.(*File); ok && file.dirty {
util.P_out("Putting " + file.Name)
file.LastVid = file.Vid
//file.Vid = filesystem.getNextVid()
file.Vid = file.ComputeVid()
file.Vtime = flushTime
file.Source = filesystem.replInfo.Pid
file.commitChunks()
if err := filesystem.database.PutFile(file); err != nil {
util.P_err("Error putting file in db: ", err)
}
if file.parent != nil {
file.parent.setChild(file)
file.parent.dirty = true
}
changes[hex.EncodeToString(file.Vid)], _ = json.Marshal(file)
}
}
// Creates a file or directory archive if possible, returns with an error
// if there is more than one "@" in name, or no file/directory exists to be
// archived
func (dir *Directory) mkArchive(name string) (*Directory, error) {
tmp := strings.Split(name, "@")
if len(tmp) != 2 || dir.children[tmp[0]] == nil {
return nil, fuse.ENOENT
}
if tmp[1] == "archive" { // File archives
node := dir.children[tmp[0]]
if file, ok := node.(*File); ok {
return dir.mkFileArchive(name, file)
}
} else { // Directory archives
archiveTime := time.Now()
if duration, err := time.ParseDuration(tmp[1]); err == nil {
archiveTime = archiveTime.Add(duration)
} else {
archiveTime, _ = util.ParseTime(tmp[1])
}
node := dir.children[tmp[0]]
util.P_out(archiveTime.String())
if subdir, ok := node.(*Directory); ok {
return dir.mkDirectoryArchive(name, archiveTime, subdir)
}
}
return nil, fuse.ENOENT
}
// Returns the File's data array
func (file *File) ReadAll(intr fs.Intr) ([]byte, fuse.Error) {
filesystem.Lock(file)
defer filesystem.Unlock(file)
util.P_out("read all: %s", file.Name)
if !file.loaded {
file.loadChunks()
}
file.Attrs.Atime = time.Now()
return file.data, nil
}
// Removes a file named req.Name from dir if it exists
func (dir *Directory) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error {
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out(req.String())
if _, ok := dir.children[req.Name]; ok {
dir.removeChild(req.Name)
dir.dirty = true
return nil
}
return fuse.ENOENT
}
// Checks for a file called name in dir and returns it if it exists
// Loads children lazily if they aren't in memory
func (dir *Directory) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) {
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out("Lookup on %q from %q\n", name, dir.Name)
if !dir.childrenInMemory {
dir.loadChildren()
}
if file, ok := dir.children[name]; ok {
return file, nil
}
return nil, fuse.ENOENT
}
func (fsys *FS) PeriodicFlush() {
for {
util.P_out("Flushing")
fsys.Lock(nil)
FlushNode(fsys.root)
if val, err := json.Marshal(filesystem); err == nil {
fsys.database.PutBlock([]byte("metadata"), val)
}
fsys.Unlock(nil)
time.Sleep(FLUSHER_PERIOD)
}
}
// Initializes and returns a filesystem with the database provided
// Note: This MUST be called before some node methods are called
// Because unfortunately for the time being, it has to set up some
// global state.
func NewFs(db FsDatabase, us *ReplicaInfo, replicas map[string]*ReplicaInfo) FS {
newFs := FS{
database: db,
NextInd: 0,
NextVid: 1,
ChunkInDb: make(map[string]bool),
}
if data, err := db.GetBlock([]byte("metadata")); err == nil {
if jsonErr := json.Unmarshal(data, &filesystem); jsonErr != nil {
filesystem = newFs
} else {
filesystem.database = db
}
} else {
filesystem = newFs
}
if root, err := filesystem.database.GetRoot(); err != nil || root == nil {
util.P_out("Couldn't find root in db, making new one")
filesystem.root = new(Directory)
filesystem.root.InitDirectory("", os.ModeDir|0755, nil)
} else {
filesystem.root = root
}
filesystem.comm, _ = NewCommunicator(us.Name, replicas)
if filesystem.comm != nil {
go filesystem.comm.MetaBroadcastListener(func(payload []byte) {
changes := make(map[string][]byte)
i := 0
for i = range payload {
if payload[i] == byte(' ') {
break
}
}
if i >= len(payload) {
return
}
json.Unmarshal(payload[i+1:], &changes)
filesystem.Lock(nil)
defer filesystem.Unlock(nil)
filesystem.updateFromBroadcast(changes)
})
go filesystem.comm.ChunkRequestListener(func(sha []byte) []byte {
data, _ := filesystem.database.GetBlock(sha)
return data
})
}
filesystem.replInfo = *us
//filesystem.replInfo = *replicas[pid]
return filesystem
}
// Makes a directory called req.Name in dir
func (dir *Directory) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) {
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out(req.String())
if strings.Contains(req.Name, "@") {
return dir.mkArchive(req.Name)
}
subdir := new(Directory)
subdir.InitDirectory(req.Name, os.ModeDir|req.Mode, dir)
dir.setChild(subdir)
subdir.dirty = true
dir.dirty = true
return subdir, nil
}
// Creates a regular file in dir with the attributes supplied in req
func (dir *Directory) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) {
if strings.Contains(req.Name, "@") {
return nil, nil, fuse.EPERM
}
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out(req.String())
rval := new(File)
rval.InitFile(req.Name, req.Mode, dir)
rval.Attrs.Gid = req.Gid
rval.Attrs.Uid = req.Uid
dir.setChild(rval)
resp.Attr = rval.Attr()
resp.Node = fuse.NodeID(rval.Attr().Inode)
rval.dirty = true
dir.dirty = true
return rval, rval, nil
}
// Returns a list of Nodes (Files or Directories) in dir
// Loads children lazily if they aren't in memory
func (dir *Directory) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) {
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out("readdir %q\n", dir.Name)
if !dir.childrenInMemory {
dir.loadChildren()
}
files := make([]fuse.Dirent, 0, 10)
files = append(files, fuse.Dirent{Inode: dir.Attr().Inode, Name: ".", Type: fuse.DT_Dir})
parent := dir.parent
if parent == nil {
parent = dir
}
files = append(files, fuse.Dirent{Inode: parent.Attr().Inode, Name: "..", Type: fuse.DT_Dir})
for name, file := range dir.children {
files = append(files, fuse.Dirent{Inode: file.Attr().Inode, Name: name, Type: fuseType(file)})
}
return files, nil
}
func (file *File) commitChunks() {
chunker := util.DefaultChunker()
chunks := chunker.Chunks(file.data)
file.DataBlocks = make([][]byte, len(chunks))
for i, chunk := range chunks {
hasher := sha1.New()
hasher.Write(chunk)
dataHash := hasher.Sum(nil)
// Academic assumption: collisions aren't a thing, so we'll assume there will
// never be any corruption, might as well save on writes in the process.
util.P_out(string(dataHash))
file.DataBlocks[i] = dataHash
if !filesystem.DbContains(dataHash) {
if dbErr := filesystem.PutChunk(dataHash, chunk); dbErr != nil {
util.P_err("Failed to write chunk to db: ", dbErr)
}
}
}
}
// Moves a file from dir to newDir (potentially the same as dir) and changes its name from req.OldName
// to req.NewName
func (dir *Directory) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fuse.Error {
filesystem.Lock(dir)
defer filesystem.Unlock(dir)
util.P_out(req.String())
if d, newDirOk := newDir.(*Directory); newDirOk {
if v, oldNameInDir := dir.children[req.OldName]; oldNameInDir {
v.setName(req.NewName)
d.setChild(v)
if file, ok := v.(*File); ok {
file.dirty = true
file.parent = d
}
dir.removeChild(req.OldName)
d.dirty = true
dir.dirty = true
return nil
}
return fuse.ENOENT
}
return fuse.Errno(syscall.ENOTDIR)
}
// Copies data from req.Data to the File's data array beginning at index req.Offset.
// Grows file.data if necessary.
func (file *File) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error {
filesystem.Lock(file)
defer filesystem.Unlock(file)
util.P_out(req.String())
writeSize := len(req.Data)
size := uint64(req.Offset) + uint64(writeSize)
if size < file.Attr().Size {
size = file.Attr().Size
}
if size > uint64(cap(file.data)) {
tmp := file.data
file.data = make([]byte, size, size*2)
copy(file.data, tmp)
} else {
file.data = file.data[:size]
}
copy(file.data[req.Offset:], req.Data)
resp.Size = writeSize
file.Attrs.Size = uint64(size)
file.Attrs.Mtime = time.Now()
file.dirty = true
return nil
}
func (fsys *FS) getNextVid() uint64 {
util.P_out(strconv.FormatUint(fsys.NextVid, 10))
currVid := fsys.NextVid
fsys.NextVid++
return currVid
}
func main() {
flag.Usage = Usage
debugPtr := flag.Bool("debug", false, "print lots of stuff")
newfsPtr := flag.Bool("newfs", false, "start with an empty file system")
mtimePtr := flag.Bool("mtimeArchives", false, "use modify timestamp instead of version timestamp for archives")
name := flag.String("name", "auto", "replica name")
configFile := flag.String("config", "config.txt", "path to config file")
flag.Parse()
util.SetDebug(*debugPtr)
myfs.UseMtime = *mtimePtr
util.P_out("main\n")
pid := myfs.GetOurPid(*configFile, *name)
replicas := myfs.ReadReplicaInfo(*configFile)
thisReplica := replicas[pid]
if thisReplica == nil {
util.P_err("No applicable replica")
os.Exit(1)
}
if *newfsPtr {
os.RemoveAll(thisReplica.DbPath)
}
if _, err := os.Stat(thisReplica.MntPoint); os.IsNotExist(err) {
os.MkdirAll(thisReplica.MntPoint, os.ModeDir)
}
db, err := myfs.NewLeveldbFsDatabase(thisReplica.DbPath)
//db := &myfs.DummyFsDb{}
//err := error(nil)
if err != nil {
util.P_err("Problem loading the database: ", err)
os.Exit(-1)
}
filesystem := myfs.NewFs(db, thisReplica, replicas)
go filesystem.PeriodicFlush()
//mountpoint := flag.Arg(0)
mountpoint := thisReplica.MntPoint
fuse.Unmount(mountpoint) //!!
c, err := fuse.Mount(mountpoint)
if err != nil {
log.Fatal(err)
}
defer c.Close()
err = fs.Serve(c, filesystem)
if err != nil {
log.Fatal(err)
}
// check if the mount process has an error to report
<-c.Ready
if err := c.MountError; err != nil {
log.Fatal(err)
}
}
func (fsys FS) Root() (fs.Node, fuse.Error) {
util.P_out("root returns as %d\n", int(fsys.root.Attr().Inode))
return fsys.root, nil
}