// Lazy file loading
func (dir *Directory) loadChildren() {
	util.P_out("Loading children")
	for key := range dir.ChildVids {
		util.P_out(key)
		if dir.IsDir[key] {
			child, err := filesystem.GetDirectory(dir.ChildVids[key], dir.Source)
			if err != nil {
				util.P_err("Error loading directory from db: ", err)
			} else {
				child.parent = dir
				child.archive = dir.isArchive()
				dir.setChild(child)
			}
		} else {
			child, err := filesystem.GetFile(dir.ChildVids[key], dir.Source)
			if err != nil {
				util.P_err("Error loading file from db: ", err)
			} else {
				child.parent = dir
				child.archive = dir.isArchive()
				dir.setChild(child)
			}
		}
	}
	dir.childrenInMemory = true
}
Пример #2
0
func (node *Node) InitNode(name string, mode os.FileMode, parent *Directory) {
	node.Source = filesystem.replInfo.Pid
	node.Vid = NULL_VERSION
	node.LastVid = NULL_VERSION
	util.P_out("VID: ", node.Vid)
	node.Attrs.Inode = filesystem.getNextInd()
	node.Attrs.Nlink = 1
	node.Name = name

	tm := time.Now()
	node.Attrs.Atime = tm
	node.Attrs.Mtime = tm
	node.Attrs.Ctime = tm
	node.Attrs.Crtime = tm
	node.Attrs.Mode = mode

	node.Attrs.Gid = uint32(gid)
	node.Attrs.Uid = uint32(uid)

	node.Attrs.Size = 0

	node.dirty = true
	node.parent = parent

	node.archive = false

	util.P_out("inited node inode %d, %q\n", filesystem.NextInd, name)
}
Пример #3
0
func flushNode(flushTime time.Time, node NamedNode, changes map[string][]byte) {
	if dir, ok := node.(*Directory); ok {
		util.P_out("Visiting node: "+dir.Name+" at version %d", dir.Vid)
		for _, node := range dir.children {
			if node != nil {
				if !node.isArchive() {
					flushNode(flushTime, node, changes)
				}
			}
		}
		// TODO: make this a dir method or another function here
		if dir.dirty {
			util.P_out("Putting " + dir.Name)
			dir.LastVid = dir.Vid
			//dir.Vid = filesystem.getNextVid()
			dir.Vid = dir.ComputeVid()
			dir.Vtime = flushTime
			dir.Source = filesystem.replInfo.Pid
			if dir.parent != nil {
				dir.parent.setChild(dir)
				if err := filesystem.database.PutDirectory(dir); err != nil {
					util.P_err("Error putting directory in db: ", err)
				}
				dir.parent.dirty = true
			} else {
				if err := filesystem.database.SetRoot(dir); err != nil {
					util.P_err("Error setting root in db: ", err)
				}
				changes[HEAD] = dir.Vid
			}
			changes[hex.EncodeToString(dir.Vid)], _ = json.Marshal(dir)
		}
	}
	if file, ok := node.(*File); ok {
		util.P_out("Visiting " + file.Name)
	}
	// TODO: make this a file method or another function here
	if file, ok := node.(*File); ok && file.dirty {
		util.P_out("Putting " + file.Name)
		file.LastVid = file.Vid
		//file.Vid = filesystem.getNextVid()
		file.Vid = file.ComputeVid()
		file.Vtime = flushTime
		file.Source = filesystem.replInfo.Pid
		file.commitChunks()
		if err := filesystem.database.PutFile(file); err != nil {
			util.P_err("Error putting file in db: ", err)
		}
		if file.parent != nil {
			file.parent.setChild(file)
			file.parent.dirty = true
		}
		changes[hex.EncodeToString(file.Vid)], _ = json.Marshal(file)
	}
}
// Creates a file or directory archive if possible, returns with an error
// if there is more than one "@" in name, or no file/directory exists to be
// archived
func (dir *Directory) mkArchive(name string) (*Directory, error) {
	tmp := strings.Split(name, "@")
	if len(tmp) != 2 || dir.children[tmp[0]] == nil {
		return nil, fuse.ENOENT
	}
	if tmp[1] == "archive" { // File archives
		node := dir.children[tmp[0]]
		if file, ok := node.(*File); ok {
			return dir.mkFileArchive(name, file)
		}
	} else { // Directory archives
		archiveTime := time.Now()
		if duration, err := time.ParseDuration(tmp[1]); err == nil {
			archiveTime = archiveTime.Add(duration)
		} else {
			archiveTime, _ = util.ParseTime(tmp[1])
		}
		node := dir.children[tmp[0]]
		util.P_out(archiveTime.String())
		if subdir, ok := node.(*Directory); ok {
			return dir.mkDirectoryArchive(name, archiveTime, subdir)
		}
	}
	return nil, fuse.ENOENT
}
Пример #5
0
// Returns the File's data array
func (file *File) ReadAll(intr fs.Intr) ([]byte, fuse.Error) {
	filesystem.Lock(file)
	defer filesystem.Unlock(file)
	util.P_out("read all: %s", file.Name)
	if !file.loaded {
		file.loadChunks()
	}
	file.Attrs.Atime = time.Now()
	return file.data, nil
}
// Removes a file named req.Name from dir if it exists
func (dir *Directory) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error {
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out(req.String())
	if _, ok := dir.children[req.Name]; ok {
		dir.removeChild(req.Name)
		dir.dirty = true
		return nil
	}
	return fuse.ENOENT
}
// Checks for a file called name in dir and returns it if it exists
// Loads children lazily if they aren't in memory
func (dir *Directory) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) {
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out("Lookup on %q from %q\n", name, dir.Name)
	if !dir.childrenInMemory {
		dir.loadChildren()
	}
	if file, ok := dir.children[name]; ok {
		return file, nil
	}
	return nil, fuse.ENOENT
}
Пример #8
0
func (fsys *FS) PeriodicFlush() {
	for {
		util.P_out("Flushing")
		fsys.Lock(nil)
		FlushNode(fsys.root)
		if val, err := json.Marshal(filesystem); err == nil {
			fsys.database.PutBlock([]byte("metadata"), val)
		}
		fsys.Unlock(nil)
		time.Sleep(FLUSHER_PERIOD)
	}
}
Пример #9
0
// Initializes and returns a filesystem with the database provided
// Note: This MUST be called before some node methods are called
// Because unfortunately for the time being, it has to set up some
// global state.
func NewFs(db FsDatabase, us *ReplicaInfo, replicas map[string]*ReplicaInfo) FS {
	newFs := FS{
		database:  db,
		NextInd:   0,
		NextVid:   1,
		ChunkInDb: make(map[string]bool),
	}
	if data, err := db.GetBlock([]byte("metadata")); err == nil {
		if jsonErr := json.Unmarshal(data, &filesystem); jsonErr != nil {
			filesystem = newFs
		} else {
			filesystem.database = db
		}
	} else {
		filesystem = newFs
	}
	if root, err := filesystem.database.GetRoot(); err != nil || root == nil {
		util.P_out("Couldn't find root in db, making new one")
		filesystem.root = new(Directory)
		filesystem.root.InitDirectory("", os.ModeDir|0755, nil)
	} else {
		filesystem.root = root
	}
	filesystem.comm, _ = NewCommunicator(us.Name, replicas)
	if filesystem.comm != nil {
		go filesystem.comm.MetaBroadcastListener(func(payload []byte) {
			changes := make(map[string][]byte)
			i := 0
			for i = range payload {
				if payload[i] == byte(' ') {
					break
				}
			}
			if i >= len(payload) {
				return
			}
			json.Unmarshal(payload[i+1:], &changes)
			filesystem.Lock(nil)
			defer filesystem.Unlock(nil)
			filesystem.updateFromBroadcast(changes)
		})
		go filesystem.comm.ChunkRequestListener(func(sha []byte) []byte {
			data, _ := filesystem.database.GetBlock(sha)
			return data
		})
	}
	filesystem.replInfo = *us
	//filesystem.replInfo = *replicas[pid]
	return filesystem
}
Пример #10
0
// Makes a directory called req.Name in dir
func (dir *Directory) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) {
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out(req.String())
	if strings.Contains(req.Name, "@") {
		return dir.mkArchive(req.Name)
	}
	subdir := new(Directory)
	subdir.InitDirectory(req.Name, os.ModeDir|req.Mode, dir)
	dir.setChild(subdir)
	subdir.dirty = true
	dir.dirty = true
	return subdir, nil
}
Пример #11
0
// Creates a regular file in dir with the attributes supplied in req
func (dir *Directory) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) {
	if strings.Contains(req.Name, "@") {
		return nil, nil, fuse.EPERM
	}
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out(req.String())
	rval := new(File)
	rval.InitFile(req.Name, req.Mode, dir)
	rval.Attrs.Gid = req.Gid
	rval.Attrs.Uid = req.Uid
	dir.setChild(rval)
	resp.Attr = rval.Attr()
	resp.Node = fuse.NodeID(rval.Attr().Inode)
	rval.dirty = true
	dir.dirty = true
	return rval, rval, nil
}
Пример #12
0
// Returns a list of Nodes (Files or Directories) in dir
// Loads children lazily if they aren't in memory
func (dir *Directory) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) {
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out("readdir %q\n", dir.Name)
	if !dir.childrenInMemory {
		dir.loadChildren()
	}
	files := make([]fuse.Dirent, 0, 10)
	files = append(files, fuse.Dirent{Inode: dir.Attr().Inode, Name: ".", Type: fuse.DT_Dir})
	parent := dir.parent
	if parent == nil {
		parent = dir
	}
	files = append(files, fuse.Dirent{Inode: parent.Attr().Inode, Name: "..", Type: fuse.DT_Dir})
	for name, file := range dir.children {
		files = append(files, fuse.Dirent{Inode: file.Attr().Inode, Name: name, Type: fuseType(file)})
	}
	return files, nil
}
Пример #13
0
func (file *File) commitChunks() {
	chunker := util.DefaultChunker()
	chunks := chunker.Chunks(file.data)
	file.DataBlocks = make([][]byte, len(chunks))
	for i, chunk := range chunks {
		hasher := sha1.New()
		hasher.Write(chunk)
		dataHash := hasher.Sum(nil)
		// Academic assumption: collisions aren't a thing, so we'll assume there will
		// never be any corruption, might as well save on writes in the process.
		util.P_out(string(dataHash))
		file.DataBlocks[i] = dataHash
		if !filesystem.DbContains(dataHash) {
			if dbErr := filesystem.PutChunk(dataHash, chunk); dbErr != nil {
				util.P_err("Failed to write chunk to db: ", dbErr)
			}
		}
	}
}
Пример #14
0
// Moves a file from dir to newDir (potentially the same as dir) and changes its name from req.OldName
// to req.NewName
func (dir *Directory) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fuse.Error {
	filesystem.Lock(dir)
	defer filesystem.Unlock(dir)
	util.P_out(req.String())
	if d, newDirOk := newDir.(*Directory); newDirOk {
		if v, oldNameInDir := dir.children[req.OldName]; oldNameInDir {
			v.setName(req.NewName)
			d.setChild(v)
			if file, ok := v.(*File); ok {
				file.dirty = true
				file.parent = d
			}
			dir.removeChild(req.OldName)
			d.dirty = true
			dir.dirty = true
			return nil
		}
		return fuse.ENOENT
	}
	return fuse.Errno(syscall.ENOTDIR)
}
Пример #15
0
// Copies data from req.Data to the File's data array beginning at index req.Offset.
// Grows file.data if necessary.
func (file *File) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error {
	filesystem.Lock(file)
	defer filesystem.Unlock(file)
	util.P_out(req.String())
	writeSize := len(req.Data)
	size := uint64(req.Offset) + uint64(writeSize)
	if size < file.Attr().Size {
		size = file.Attr().Size
	}
	if size > uint64(cap(file.data)) {
		tmp := file.data
		file.data = make([]byte, size, size*2)
		copy(file.data, tmp)
	} else {
		file.data = file.data[:size]
	}
	copy(file.data[req.Offset:], req.Data)
	resp.Size = writeSize
	file.Attrs.Size = uint64(size)
	file.Attrs.Mtime = time.Now()
	file.dirty = true
	return nil
}
Пример #16
0
func (fsys *FS) getNextVid() uint64 {
	util.P_out(strconv.FormatUint(fsys.NextVid, 10))
	currVid := fsys.NextVid
	fsys.NextVid++
	return currVid
}
Пример #17
0
func main() {
	flag.Usage = Usage

	debugPtr := flag.Bool("debug", false, "print lots of stuff")
	newfsPtr := flag.Bool("newfs", false, "start with an empty file system")
	mtimePtr := flag.Bool("mtimeArchives", false, "use modify timestamp instead of version timestamp for archives")
	name := flag.String("name", "auto", "replica name")
	configFile := flag.String("config", "config.txt", "path to config file")
	flag.Parse()
	util.SetDebug(*debugPtr)
	myfs.UseMtime = *mtimePtr

	util.P_out("main\n")

	pid := myfs.GetOurPid(*configFile, *name)
	replicas := myfs.ReadReplicaInfo(*configFile)
	thisReplica := replicas[pid]

	if thisReplica == nil {
		util.P_err("No applicable replica")
		os.Exit(1)
	}

	if *newfsPtr {
		os.RemoveAll(thisReplica.DbPath)
	}

	if _, err := os.Stat(thisReplica.MntPoint); os.IsNotExist(err) {
		os.MkdirAll(thisReplica.MntPoint, os.ModeDir)
	}

	db, err := myfs.NewLeveldbFsDatabase(thisReplica.DbPath)
	//db := &myfs.DummyFsDb{}
	//err := error(nil)
	if err != nil {
		util.P_err("Problem loading the database: ", err)
		os.Exit(-1)
	}
	filesystem := myfs.NewFs(db, thisReplica, replicas)
	go filesystem.PeriodicFlush()

	//mountpoint := flag.Arg(0)
	mountpoint := thisReplica.MntPoint

	fuse.Unmount(mountpoint) //!!
	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	err = fs.Serve(c, filesystem)
	if err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}
Пример #18
0
func (fsys FS) Root() (fs.Node, fuse.Error) {
	util.P_out("root returns as %d\n", int(fsys.root.Attr().Inode))
	return fsys.root, nil
}