// Retrieves a directory in one go.
// Also used to check existence; returning ds.ErrNoSuchEntity
func (fs *dsFileSys) dirByPath(name string) (DsDir, error) {
dir, bname := fs.SplitX(name)
fo := DsDir{}
fo.fSys = fs
preciseK := ds.NewKey(fs.c, tdir, dir+common.Filify(bname), 0, nil)
fo.Key = preciseK
err := fo.MemCacheGet(dir + common.Filify(bname))
if err == nil {
fo.fSys = fs
fo.Key = preciseK
// log.Printf(" mcg %-16v %v", dir+bname, fo.Key)
return fo, nil
}
err = ds.Get(fs.c, preciseK, &fo)
if err == ds.ErrNoSuchEntity {
// uncomment to see where directories do not exist:
// log.Printf("no directory: %-20v ", path)
// runtimepb.StackTrace(4)
return fo, err
} else if err != nil {
aelog.Errorf(fs.Ctx(), "Error getting dir %v => %v", dir+common.Filify(bname), err)
}
fo.MemCacheSet()
return fo, err
}
func (fs *dsFileSys) saveDirByPathExt(dirObj DsDir, name string) (DsDir, error) {
fo := DsDir{}
fo.isDir = true
fo.MModTime = dirObj.MModTime
if dirObj.MMode == 0 {
fo.MMode = 0755
} else {
fo.MMode = dirObj.MMode
}
if dirObj.MModTime.IsZero() {
fo.MModTime = time.Now()
} else {
fo.MModTime = dirObj.MModTime
}
fo.fSys = fs
dir, bname := fs.SplitX(name)
fo.Dir = dir
fo.BName = common.Filify(bname)
preciseK := ds.NewKey(fs.c, tdir, dir+common.Filify(bname), 0, nil)
fo.Key = preciseK
// fs.Ctx().Infof("Saving dir %-14q %q %v ", fo.Dir, fo.BName, fo.Key)
effKey, err := ds.Put(fs.c, preciseK, &fo)
if err != nil {
aelog.Errorf(fs.Ctx(), "Error saving dir %v => %v", dir+bname, err)
return fo, err
}
if !preciseK.Equal(effKey) {
aelog.Errorf(fs.Ctx(), "dir keys unequal %v - %v", preciseK, effKey)
}
fo.MemCacheSet()
// recurse upwards
_, err = fs.dirByPath(fo.Dir)
if err == ds.ErrNoSuchEntity {
_, err = fs.saveDirByPath(fo.Dir)
if err != nil {
return fo, err
}
}
return fo, nil
}
// similar to ReadDir but returning only files
func (fs *dsFileSys) filesByPath(name string) ([]DsFile, error) {
dir, bname := fs.SplitX(name)
var files []DsFile
foDir, err := fs.dirByPath(dir + common.Filify(bname))
if err == datastore.ErrNoSuchEntity {
return files, err
} else if err != nil {
aelog.Errorf(fs.Ctx(), "Error reading dir for files %v => %v", dir+bname, err)
return files, err
}
// fs.Ctx().Infof(" Files by Path %-20v - got dir %-10v - %v", dir+bname, foDir.Name(), foDir.Key)
q := datastore.NewQuery(tfil).Ancestor(foDir.Key)
keys, err := q.GetAll(fs.Ctx(), &files)
if err != nil {
aelog.Errorf(fs.Ctx(), "Error fetching files children of %v => %v", foDir.Key, err)
return files, err
}
// fs.Ctx().Infof(" Files by Path %-20v - got files %v", dir+bname, len(files))
for i := 0; i < len(files); i++ {
files[i].Key = keys[i]
files[i].fSys = fs
}
fs.filesorter(files)
return files, err
}
// subtreeByPath retrieves a subdirectories of a given directory.
// It is relying on an indexed string property "Dir"
// containing a string representation of the full path.
//
// It might be fast for deep, uncached directory subtree,
// that have been saved in nested manner.
//
// However, it might not find recently added directories.
// Upon finding nothing, it therefore returns the
// "warning" fsi.EmptyQueryResult
//
// The func could easily be enhanced chunked scanning.
//
// It is currently used by ReadDir and by the test package.
// It is public for the test package.
func (fs *dsFileSys) SubtreeByPath(name string, onlyDirectChildren bool) ([]DsDir, error) {
dir, bname := fs.SplitX(name)
name = dir + common.Filify(bname)
if !strings.HasSuffix(name, sep) {
name += sep
}
var q *datastore.Query
if onlyDirectChildren {
q = datastore.NewQuery(tdir).
Filter("Dir=", name).
Order("Dir")
// Limit(4)
} else {
pathInc := IncrementString(name)
q = datastore.NewQuery(tdir).
Filter("Dir>=", name).
Filter("Dir<", pathInc).
Order("Dir")
}
// log.Printf("%v", q)
var children []DsDir
keys, err := q.GetAll(fs.Ctx(), &children)
if err != nil {
aelog.Errorf(fs.Ctx(), "Error getting all children of %v => %v", dir+bname, err)
return children, err
}
if len(children) < 1 {
return children, fsi.EmptyQueryResult
}
// Very evil: We filter out root node, since it's
// has the same dir as the level-1 directories.
keyRoot := datastore.NewKey(fs.Ctx(), tdir, fs.RootDir(), 0, nil)
idxRoot := -1
for i := 0; i < len(children); i++ {
children[i].fSys = fs
children[i].Key = keys[i]
if keys[i].Equal(keyRoot) {
// log.Printf("root idx %v", i)
idxRoot = i
}
}
if idxRoot > -1 {
children = append(children[:idxRoot], children[idxRoot+1:]...)
}
return children, nil
}
func (m *memMapFs) Open(name string) (fsi.File, error) {
origName := name
dir, bname := m.SplitX(name)
name = dir + bname // not join, since it removes trailing slash
name1 := name
name2 := name
if strings.HasSuffix(name, "/") {
// explicitly asked for dir
name2 = common.Filify(name)
} else {
// try file first, then try the directory
name2 = common.Directorify(name)
}
m.rlock()
f, ok := m.fos[name1]
if !ok {
f, ok = m.fos[name2]
}
if ok {
ff, okConv := f.(*InMemoryFile)
if okConv {
ff.Open()
} else {
return nil, fmt.Errorf("could not convert opened file into InMemoryFile 1")
}
}
m.runlock()
//
//
// Fallback to underlying fs
if !ok {
var err error
f, err = m.lookupUnderlyingFS(name, origName)
if err != nil {
// log.Printf("underlying says %q => %v\n", name, err)
return nil, err
return nil, fsi.ErrFileNotFound
}
// log.Printf("underlying succ %q\n", name)
// m.Dump()
return f, nil
}
//
// Regular return
if ok {
return f, nil
} else {
return nil, fsi.ErrFileNotFound
}
}
//
//
// Path is the directory, BName contains the base name.
func (fs *dsFileSys) saveFileByPath(f *DsFile, name string) error {
dir, bname := fs.SplitX(name)
f.Dir = dir
// bname was only submitted in the fileobject
// correct previous
if f.BName != "" && f.BName != bname {
dir = dir + bname // re-append
if !strings.HasSuffix(dir, sep) {
dir += sep
}
bname = f.BName
f.Dir = dir
}
f.BName = common.Filify(bname)
// f.MModTime = time.Now()
f.fSys = fs
//
// -------------now the datastore part-------------------------
foDir, err := fs.dirByPath(dir)
if err == datastore.ErrNoSuchEntity {
return err
} else if err != nil {
aelog.Errorf(fs.Ctx(), "Error reading dir for file %v => %v", dir+bname, err)
return err
}
suggKey := datastore.NewKey(fs.Ctx(), tfil, f.BName, 0, foDir.Key)
f.Key = suggKey
effKey, err := datastore.Put(fs.Ctx(), suggKey, f)
if err != nil {
aelog.Errorf(fs.Ctx(), "Error saving file %v => %v", dir+bname, err)
return err
}
if !suggKey.Equal(effKey) {
aelog.Errorf(fs.Ctx(), "file keys unequal %v - %v; %v %s", suggKey, effKey, f.Dir+f.BName, f.Data)
runtimepb.StackTrace(6)
}
// f.MemCacheSet()
return nil
}
// remove f from it's parent's directory
func (m *memMapFs) unRegisterWithParent(name string) error {
parent, bname := m.SplitX(name)
name = parent + bname
pDir, err := m.findParent(name)
if err != nil {
return err
}
pDirC := pDir.(*InMemoryFile)
// log.Printf("trying to unregister %-22q in %q - \n\t%+v\n", name, pDirC.name, pDirC.memDir)
delete(pDirC.memDir, common.Directorify(name))
delete(pDirC.memDir, common.Filify(name))
return nil
}
// Create opens for read-write.
// Open opens for readonly access.
func (fs *dsFileSys) Create(name string) (fsi.File, error) {
// WriteFile & Create
dir, bname := fs.SplitX(name)
f := DsFile{}
f.fSys = fs
f.BName = common.Filify(bname)
f.Dir = dir
f.MModTime = time.Now()
f.MMode = 0644
// let all the properties by set by fs.saveFileByPath
err := f.Sync()
if err != nil {
return nil, err
}
// return &f, nil
ff := fsi.File(&f)
return ff, err
}
// Only one save operation required
func (fs *dsFileSys) WriteFile(name string, data []byte, perm os.FileMode) error {
// WriteFile & Create
dir, bname := fs.SplitX(name)
f := DsFile{}
f.Dir = dir
f.BName = common.Filify(bname)
f.fSys = fs
f.MModTime = time.Now()
var err error
_, err = f.Write(data)
if err != nil {
return err
}
err = f.Sync()
if err != nil {
return err
}
return nil
}