func (this *Fs) ListX(frominode string) ([]*filetype.KvmapEntry, error) { var fd = dvc.GetFD(GenFileName(frominode, FOLDER_MAP), this.io) if fd == nil { Secretary.Error("kernel.filesystem::List", "Fail to get FD for "+frominode) return nil, exception.EX_IO_ERROR } defer fd.Release() fd.GraspReader() defer fd.ReleaseReader() if err := fd.Sync(); err != nil { Secretary.Error("kernel.filesystem::List", "SYNC error for "+frominode+": "+err.Error()) } if content, err := fd.Read(); err != nil { Secretary.Error("kernel.filesystem::List", "Read error for "+frominode+": "+err.Error()) return nil, err } else { var ret = []*filetype.KvmapEntry{} for k, v := range content { if CheckValidFilename(k) { ret = append(ret, v) } } return ret, nil } }
// Format the filesystem. // TODO: Setup clear old fs info? Up to now set up will not clear old data and will not // remove the old folder map func (this *Fs) FormatFS() error { var initMetaSelf = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: this.rootName, }) if err := this.io.Put(GenFileName(this.rootName, ".."), filetype.NewNnode(this.rootName), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::FormatFS", "Fail to create .. link for Root.") return err } if err := this.io.Put(GenFileName(this.rootName, "."), filetype.NewNnode(this.rootName), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::FormatFS", "Fail to create . link for Root.") return err } { var rootFD = dvc.GetFD(GenFileName(this.rootName, FOLDER_MAP), this.io) if rootFD == nil { Secretary.Error("kernel.filesystem::FormatFS", "Fail to get FD for Root.") return exception.EX_IO_ERROR } if err := rootFD.Submit(fc.FastMakeFolderPatch(".", "..")); err != nil { Secretary.Error("kernel.filesystem::FormatFS", "Fail to submit format patch for Root.") rootFD.Release() return nil } rootFD.Release() } // setup Trash for users return this.Mkdir(TRASH_BOX, this.rootName, true) }
func (this *Fs) BatchPutDir(filenameprefix string, frominode string, fromn int, ton int, content string) error { var kvmp = filetype.NewKvMap() var nowTime = GetTimestamp() kvmp.CheckOut() for i := fromn; i < ton; i++ { var filename = filenameprefix + strconv.Itoa(i) var nnodeName = GenFileName(frominode, filename) var newNnode = filetype.NewNnode(content) var initMeta = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: frominode, }) if err := this.io.Put(nnodeName, newNnode, initMeta); err != nil { Secretary.Error("kernel/filesystem::Fs::BatchPutDir", "Error when putting "+filename+": "+err.Error()) return err } kvmp.Kvm[filename] = &filetype.KvmapEntry{ Key: filename, Val: "", Timestamp: nowTime, } } kvmp.CheckIn() kvmp.TSet(nowTime) { var parentFolderMapFD = dvc.GetFD(GenFileName(frominode, FOLDER_MAP), this.io) if parentFolderMapFD == nil { Secretary.Error("kernel.filesystem::BatchPutDir", "Fail to create foldermap") return exception.EX_IO_ERROR } if err := parentFolderMapFD.Submit(kvmp); err != nil { Secretary.Error("kernel.filesystem::BatchPutDir", "Fail to submit patch to foldermap") parentFolderMapFD.Release() return err } parentFolderMapFD.Release() } return nil }
// If the file exist and forceMake==false, an error EX_FOLDER_ALREADY_EXIST will be returned func (this *Fs) MkdirParalleled(foldername string, frominode string, forceMake bool) error { if !CheckValidFilename(foldername) { return exception.EX_INVALID_FILENAME } // nnodeName: parentInode::foldername var nnodeName = GenFileName(frominode, foldername) if !forceMake { if tmeta, _ := this.io.Getinfo(nnodeName); tmeta != nil { return exception.EX_FOLDER_ALREADY_EXIST } } var newDomainname = uniqueid.GenGlobalUniqueName() var globalError *errorgroup.ErrorAssembly = nil var geLock sync.Mutex var wg sync.WaitGroup // create target inode and write parent folder map var routineToCreateInode = func() { defer wg.Done() var newNnode = filetype.NewNnode(newDomainname) var initMeta = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: frominode, }) if err := this.io.Put(nnodeName, newNnode, initMeta); err != nil { geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } // submit patch to parent folder's map { var parentFolderMapFD = dvc.GetFD(GenFileName(frominode, FOLDER_MAP), this.io) if parentFolderMapFD == nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to create foldermap fd for new folder "+nnodeName+"'s parent map'") geLock.Lock() globalError = errorgroup.AddIn(globalError, exception.EX_IO_ERROR) geLock.Unlock() return } if err := parentFolderMapFD.Submit(fc.FastMakeFolderPatch(foldername)); err != nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to submit patch to foldermap for new folder "+nnodeName+"'s parent map'") parentFolderMapFD.Release() geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } parentFolderMapFD.Release() } } var initMetaSelf = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: newDomainname, }) var routineToCreateDotDot = func() { defer wg.Done() if err := this.io.Put(GenFileName(newDomainname, ".."), filetype.NewNnode(frominode), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to create .. link for new folder "+nnodeName+".") geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } } var routineToCreateDot = func() { defer wg.Done() if err := this.io.Put(GenFileName(newDomainname, "."), filetype.NewNnode(newDomainname), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to create . link for new folder "+nnodeName+".") geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } } var routineToWriteNewMap = func() { defer wg.Done() { var newFolderMapFD = dvc.GetFD(GenFileName(newDomainname, FOLDER_MAP), this.io) if newFolderMapFD == nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to create foldermap fd for new folder "+nnodeName+".") geLock.Lock() globalError = errorgroup.AddIn(globalError, exception.EX_IO_ERROR) geLock.Unlock() return } if err := newFolderMapFD.Submit(fc.FastMakeFolderPatch(".", "..")); err != nil { Secretary.Error("kernel.filesystem::MkdirParalleled", "Fail to init foldermap for new folder "+nnodeName+".") newFolderMapFD.Release() geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } newFolderMapFD.Release() } } wg.Add(4) go routineToCreateInode() go routineToCreateDotDot() go routineToCreateDot() go routineToWriteNewMap() wg.Wait() // NOW All the routines have returned and globalError stores all the errors // TODO: consider clearing roll-back return globalError }
// Attentez: It is not atomic // If byForce set to false and the destination file exists, an EX_FOLDER_ALREADY_EXIST will be returned func (this *Fs) MvXParalleled(srcName, srcInode, desName, desInode string, byForce bool) error { //Insider.LogD("MvXParalleled start") //defer Insider.LogD("MvXParalleled end") if !CheckValidFilename(srcName) || !CheckValidFilename(desName) { return exception.EX_INVALID_FILENAME } if !byForce && outapi.ForceCheckExist(this.io.CheckExist(GenFileName(desInode, desName))) { return exception.EX_FOLDER_ALREADY_EXIST } var modifiedMeta = FileMeta(map[string]string{ META_PARENT_INODE: desInode, }) //Insider.LogD("MvXParalleled::copy start") if err := this.io.Copy(GenFileName(srcInode, srcName), GenFileName(desInode, desName), modifiedMeta); err != nil { return exception.EX_FILE_NOT_EXIST } //Insider.LogD("MvXParalleled::copy end") var globalError *errorgroup.ErrorAssembly = nil var geLock sync.Mutex var wg sync.WaitGroup var routineToUpdateDesParentMap = func(oriMeta FileMeta) { defer wg.Done() //Insider.LogD("routineToUpdateDesParentMap start") //defer Insider.LogD("routineToUpdateDesParentMap end") if oriMeta == nil { return } { var desParentMap = dvc.GetFD(GenFileName(desInode, FOLDER_MAP), this.io) //Insider.Log("routineToUpdateDesParentMap", "GOT FD") if desParentMap == nil { Secretary.Error("kernel.filesystem::MvXParalleled", "Fail to get foldermap fd for folder "+desInode) geLock.Lock() globalError = errorgroup.AddIn(globalError, exception.EX_IO_ERROR) geLock.Unlock() return } if err := desParentMap.Submit(fc.FastMakeWithMeta(desName, InferFMapMetaFromNNodeMeta(oriMeta))); err != nil { Secretary.Error("kernel.filesystem::MvXParalleled", "Fail to submit foldermap patch for folder "+desInode) desParentMap.Release() geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } //Insider.Log("routineToUpdateDesParentMap", "SUBMITTED") desParentMap.Release() //Insider.Log("routineToUpdateDesParentMap", "RELEASED") } } var routineToRemoveOldNnode = func() { defer wg.Done() //Insider.LogD("routineToRemoveOldNnode start") //defer Insider.LogD("routineToRemoveOldNnode end") this.io.Delete(GenFileName(srcInode, srcName)) } var routineToUpdateSrcParentMap = func() { defer wg.Done() //Insider.LogD("routineToUpdateSrcParentMap start") //defer Insider.LogD("routineToUpdateSrcParentMap end") { var srcParentMap = dvc.GetFD(GenFileName(srcInode, FOLDER_MAP), this.io) //Insider.Log("routineToUpdateSrcParentMap", "GOT FD") if srcParentMap == nil { Secretary.Error("kernel.filesystem::MvXParalleled", "Fail to get foldermap fd for folder "+srcInode) geLock.Lock() globalError = errorgroup.AddIn(globalError, exception.EX_IO_ERROR) geLock.Unlock() return } if err := srcParentMap.Submit(filetype.FastAntiMake(srcName)); err != nil { Secretary.Error("kernel.filesystem::MvXParalleled", "Fail to submit foldermap patch for folder "+srcInode) srcParentMap.Release() geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } //Insider.Log("routineToUpdateSrcParentMap", "SUBMITTED") srcParentMap.Release() //Insider.Log("routineToUpdateSrcParentMap", "RELEASED") } } var routineToUpdateDotDot = func() { defer wg.Done() //Insider.LogD("routineToUpdateDotDot start") //defer Insider.LogD("routineToUpdateDotDot end") var dstNnodeMeta, dstFileNnodeOriginal, _ = this.io.Get(GenFileName(desInode, desName)) go routineToUpdateDesParentMap(dstNnodeMeta) var dstFileNnode, _ = dstFileNnodeOriginal.(*filetype.Nnode) if dstFileNnode == nil { Secretary.Error("kernel.filesystem::MvX", "Fail to read nnode "+GenFileName(desInode, desName)+".") geLock.Lock() globalError = errorgroup.AddIn(globalError, exception.EX_IO_ERROR) geLock.Unlock() return } var target = GenFileName(dstFileNnode.DesName, "..") if err := this.io.Put(target, filetype.NewNnode(desInode), nil); err != nil { Secretary.Error("kernel.filesystem::MvX", "Fail to modify .. link for "+dstFileNnode.DesName+".") geLock.Lock() globalError = errorgroup.AddIn(globalError, err) geLock.Unlock() return } else { // Secretary.Log("kernel.filesystem::MvX", "Update file "+target) } } wg.Add(4) // routineToUpdateDesParentMap() is invoked in routineToUpdateDotDot() go routineToRemoveOldNnode() go routineToUpdateSrcParentMap() go routineToUpdateDotDot() wg.Wait() return globalError }
func (this *Fs) Put(filename string, frominode string, meta FileMeta /*=nil*/, dataSource io.Reader) (error, string) { var targetFileinode string var oldOriName string if filename != "" { // CREATE MODE if !CheckValidFilename(filename) { return exception.EX_INVALID_FILENAME, "" } // set inode targetFileinode = uniqueid.GenGlobalUniqueNameWithTag("Stream") } else { // OVERRIDE MODE var oldMeta, err = this.io.Getinfo(frominode) if oldMeta == nil { if err != nil { return err, "" } return exception.EX_FILE_NOT_EXIST, "" } oldOriName = oldMeta[META_ORIGINAL_NAME] targetFileinode = frominode } // set object node if meta == nil { meta = NewMeta() } meta = meta.Clone() meta[METAKEY_TYPE] = STREAM_TYPE if filename != "" { meta[META_ORIGINAL_NAME] = filename } else { meta[META_ORIGINAL_NAME] = oldOriName } if wc, err := this.io.PutStream(targetFileinode, meta); err != nil { Secretary.Error("kernel.filesystem::Put", "Put stream for new file "+GenFileName(frominode, filename)+" failed.") return err, targetFileinode } else { if _, err2 := io.Copy(wc, dataSource); err2 != nil { wc.Close() Secretary.Error("kernel.filesystem::Put", "Piping stream for new file "+GenFileName(frominode, filename)+" failed.") return err2, targetFileinode } if err2 := wc.Close(); err2 != nil { Secretary.Error("kernel.filesystem::Put", "Close writer for new file "+GenFileName(frominode, filename)+" failed.") return err2, targetFileinode } } if filename != "" { // CREATE MODE. Set its parent node's foldermap and write the nnode concurrently var currentHeader, terr = this.io.GetinfoX(targetFileinode) if currentHeader == nil { if terr != nil { Secretary.Warn("kernel.filesystem::Put", "Fail to refetch supposed-to-be file meta: "+targetFileinode+". Error is "+terr.Error()) } else { Secretary.Warn("kernel.filesystem::Put", "Fail to refetch supposed-to-be file meta: "+targetFileinode+". The file seems to be non-existence.") } return exception.EX_CONCURRENT_CHAOS, targetFileinode } //fmt.Println(currentHeader) var pointedMeta = ConvertFileHeaderToNNodeMeta(currentHeader) var metaToSet = pointedMeta metaToSet[META_PARENT_INODE] = frominode metaToSet[META_INODE_TYPE] = META_INODE_TYPE_FILE var wg = sync.WaitGroup{} var globalError *egg.ErrorAssembly = nil var geLock sync.Mutex wg.Add(2) go (func() { defer wg.Done() // Write the nnode if err := this.io.Put(GenFileName(frominode, filename), filetype.NewNnode(targetFileinode), metaToSet); err != nil { Secretary.Warn("kernel.filesystem::Put", "Put nnode for new file "+GenFileName(frominode, filename)+" failed.") geLock.Lock() globalError = egg.AddIn(globalError, err) geLock.Unlock() return } })() go (func() { // update parentNode's foldermap defer wg.Done() var parentFD = dvc.GetFD(GenFileName(frominode, FOLDER_MAP), this.io) if parentFD == nil { Secretary.Error("kernel.filesystem::Put", "Get FD for "+GenFileName(frominode, FOLDER_MAP)+" failed.") geLock.Lock() globalError = egg.AddIn(globalError, exception.EX_INDEX_ERROR) geLock.Unlock() return } if err := parentFD.Submit(fc.FastMakeWithMeta(filename, InferFMapMetaFromNNodeMeta(metaToSet))); err != nil { Secretary.Error("kernel.filesystem::Put", "Submit patch for "+GenFileName(frominode, filename)+" failed: "+err.Error()) parentFD.Release() geLock.Lock() globalError = egg.AddIn(globalError, exception.EX_INDEX_ERROR) geLock.Unlock() return } parentFD.Release() })() wg.Wait() if globalError != nil { return globalError, targetFileinode } } return nil, targetFileinode }
// Attentez: It is not atomic // If byForce set to false and the destination file exists, an EX_FOLDER_ALREADY_EXIST will be returned func (this *Fs) MvX(srcName, srcInode, desName, desInode string, byForce bool) error { // Create a mirror at destination position. // Then, remove the old one. // Third, modify the .. pointer. if !CheckValidFilename(srcName) || !CheckValidFilename(desName) { return exception.EX_INVALID_FILENAME } if !byForce && outapi.ForceCheckExist(this.io.CheckExist(GenFileName(desInode, desName))) { return exception.EX_FOLDER_ALREADY_EXIST } var modifiedMeta = FileMeta(map[string]string{ META_PARENT_INODE: desInode, }) if err := this.io.Copy(GenFileName(srcInode, srcName), GenFileName(desInode, desName), modifiedMeta); err != nil { return exception.EX_FILE_NOT_EXIST } // remove the old one. this.io.Delete(GenFileName(srcInode, srcName)) { var srcParentMap = dvc.GetFD(GenFileName(srcInode, FOLDER_MAP), this.io) if srcParentMap == nil { Secretary.Error("kernel.filesystem::MvX", "Fail to get foldermap fd for folder "+srcInode) return exception.EX_IO_ERROR } if err := srcParentMap.Submit(filetype.FastAntiMake(srcName)); err != nil { Secretary.Error("kernel.filesystem::MvX", "Fail to submit foldermap patch for folder "+srcInode) srcParentMap.Release() return err } srcParentMap.Release() } // modify the .. pointer var dstMeta, dstFileNnodeOriginal, _ = this.io.Get(GenFileName(desInode, desName)) var dstFileNnode, _ = dstFileNnodeOriginal.(*filetype.Nnode) if dstFileNnode == nil { Secretary.Error("kernel.filesystem::MvX", "Fail to read nnode "+GenFileName(desInode, desName)+".") return exception.EX_IO_ERROR } { var desParentMap = dvc.GetFD(GenFileName(desInode, FOLDER_MAP), this.io) if desParentMap == nil { Secretary.Error("kernel.filesystem::MvX", "Fail to get foldermap fd for folder "+desInode) return exception.EX_IO_ERROR } if err := desParentMap.Submit(fc.FastMakeWithMeta(desName, InferFMapMetaFromNNodeMeta(dstMeta))); err != nil { Secretary.Error("kernel.filesystem::MvX", "Fail to submit foldermap patch for folder "+desInode) desParentMap.Release() return err } desParentMap.Release() } var target = GenFileName(dstFileNnode.DesName, "..") if err := this.io.Put(target, filetype.NewNnode(desInode), nil); err != nil { Secretary.Error("kernel.filesystem::MvX", "Fail to modify .. link for "+dstFileNnode.DesName+".") return err } else { //Secretary.Log("kernel.filesystem::MvX", "Update file "+target) } // ALL DONE! return nil }
// If the file exist and forceMake==false, an error EX_FOLDER_ALREADY_EXIST will be returned func (this *Fs) Mkdir(foldername string, frominode string, forceMake bool) error { if !CheckValidFilename(foldername) { return exception.EX_INVALID_FILENAME } // nnodeName: parentInode::foldername var nnodeName = GenFileName(frominode, foldername) if !forceMake { if tmeta, _ := this.io.Getinfo(nnodeName); tmeta != nil { return exception.EX_FOLDER_ALREADY_EXIST } } // newDomainname: <GENERATED> var newDomainname = uniqueid.GenGlobalUniqueName() var newNnode = filetype.NewNnode(newDomainname) var initMeta = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: frominode, }) if err := this.io.Put(nnodeName, newNnode, initMeta); err != nil { return err } // initialize two basic element var initMetaSelf = FileMeta(map[string]string{ META_INODE_TYPE: META_INODE_TYPE_FOLDER, META_PARENT_INODE: newDomainname, }) if err := this.io.Put(GenFileName(newDomainname, ".."), filetype.NewNnode(frominode), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to create .. link for new folder "+nnodeName+".") return err } if err := this.io.Put(GenFileName(newDomainname, "."), filetype.NewNnode(newDomainname), initMetaSelf); err != nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to create . link for new folder "+nnodeName+".") return err } // write new folder's map { var newFolderMapFD = dvc.GetFD(GenFileName(newDomainname, FOLDER_MAP), this.io) if newFolderMapFD == nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to create foldermap fd for new folder "+nnodeName+".") return exception.EX_IO_ERROR } if err := newFolderMapFD.Submit(fc.FastMakeFolderPatch(".", "..")); err != nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to init foldermap for new folder "+nnodeName+".") newFolderMapFD.Release() return err } newFolderMapFD.Release() } // submit patch to parent folder's map { var parentFolderMapFD = dvc.GetFD(GenFileName(frominode, FOLDER_MAP), this.io) if parentFolderMapFD == nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to create foldermap fd for new folder "+nnodeName+"'s parent map'") return exception.EX_IO_ERROR } if err := parentFolderMapFD.Submit(fc.FastMakeFolderPatch(foldername)); err != nil { Secretary.Error("kernel.filesystem::Mkdir", "Fail to submit patch to foldermap for new folder "+nnodeName+"'s parent map'") parentFolderMapFD.Release() return err } parentFolderMapFD.Release() } return nil }