func (sub *Sub) compareEntries(request *subproto.UpdateRequest,
subEntry, requiredEntry *filesystem.DirectoryEntry, myPathName string) {
subInode := subEntry.Inode()
requiredInode := requiredEntry.Inode()
sameType, sameMetadata, sameData := filesystem.CompareInodes(
subInode, requiredInode, nil)
if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
if sameMetadata {
return
}
if sameType {
makeDirectory(request, requiredInode, myPathName, false)
} else {
makeDirectory(request, requiredInode, myPathName, true)
}
return
}
if sameType && sameData && sameMetadata {
sub.relink(request, subEntry, requiredEntry, myPathName)
return
}
if sameType && sameData {
sub.updateMetadata(request, requiredEntry, myPathName)
sub.relink(request, subEntry, requiredEntry, myPathName)
return
}
sub.addInode(request, requiredEntry, myPathName)
}
func addSymlink(dirent *filesystem.DirectoryEntry,
fileSystem, oldFS *FileSystem,
directoryPathName string, stat *syscall.Stat_t) error {
if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
if inode, ok := inode.(*filesystem.SymlinkInode); ok {
dirent.SetInode(inode)
return nil
}
return errors.New("Inode changed type: " + dirent.Name)
}
inode := makeSymlinkInode(stat)
err := scanSymlinkInode(inode, fileSystem,
path.Join(directoryPathName, dirent.Name))
if err != nil {
return err
}
if oldFS != nil && oldFS.InodeTable != nil {
if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
if oldInode, ok := oldInode.(*filesystem.SymlinkInode); ok {
if filesystem.CompareSymlinkInodes(inode, oldInode, nil) {
inode = oldInode
}
}
}
}
dirent.SetInode(inode)
fileSystem.InodeTable[stat.Ino] = inode
return nil
}
func (sub *Sub) addEntry(request *subproto.UpdateRequest,
requiredEntry *filesystem.DirectoryEntry, myPathName string) {
requiredInode := requiredEntry.Inode()
if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
makeDirectory(request, requiredInode, myPathName, true)
} else {
sub.addInode(request, requiredEntry, myPathName)
}
}
func (decoderData *decoderData) addEntry(parent *filesystem.DirectoryInode,
fullName, name string, inode filesystem.GenericInode) {
var newEntry filesystem.DirectoryEntry
newEntry.Name = name
newEntry.InodeNumber = decoderData.nextInodeNumber
newEntry.SetInode(inode)
parent.EntryList = append(parent.EntryList, &newEntry)
decoderData.addInode(fullName, inode)
}
func (sub *Sub) updateMetadata(request *subproto.UpdateRequest,
requiredEntry *filesystem.DirectoryEntry, myPathName string) {
if sub.inodesChanged[requiredEntry.InodeNumber] {
return
}
var inode subproto.Inode
inode.Name = myPathName
inode.GenericInode = requiredEntry.Inode()
request.InodesToChange = append(request.InodesToChange, inode)
sub.inodesChanged[requiredEntry.InodeNumber] = true
}
func (decoderData *decoderData) addHardlink(header *tar.Header,
parent *filesystem.DirectoryInode, name string) error {
header.Linkname = normaliseFilename(header.Linkname)
if inum, ok := decoderData.inodeTable[header.Linkname]; ok {
var newEntry filesystem.DirectoryEntry
newEntry.Name = name
newEntry.InodeNumber = inum
parent.EntryList = append(parent.EntryList, &newEntry)
} else {
return errors.New(fmt.Sprintf("missing hardlink target: %s",
header.Linkname))
}
return nil
}
func setComputedFileMtime(requiredInode *filesystem.RegularInode,
subEntry *filesystem.DirectoryEntry) {
if requiredInode.MtimeSeconds >= 0 {
return
}
if subEntry != nil {
subInode := subEntry.Inode()
if subInode, ok := subInode.(*filesystem.RegularInode); ok {
if requiredInode.Hash == subInode.Hash {
requiredInode.MtimeNanoSeconds = subInode.MtimeNanoSeconds
requiredInode.MtimeSeconds = subInode.MtimeSeconds
return
}
}
}
requiredInode.MtimeSeconds = time.Now().Unix()
}
func compareDirectories(request *subproto.UpdateRequest, state *state,
subDirectory, requiredDirectory *filesystem.DirectoryInode,
myPathName string, filter *filter.Filter) {
// First look for entries that should be deleted.
if subDirectory != nil {
for name := range subDirectory.EntriesByName {
pathname := path.Join(myPathName, name)
if filter.Match(pathname) {
continue
}
if _, ok := requiredDirectory.EntriesByName[name]; !ok {
request.PathsToDelete = append(request.PathsToDelete, pathname)
}
}
}
for name, requiredEntry := range requiredDirectory.EntriesByName {
pathname := path.Join(myPathName, name)
if filter.Match(pathname) {
continue
}
var subEntry *filesystem.DirectoryEntry
if subDirectory != nil {
if se, ok := subDirectory.EntriesByName[name]; ok {
subEntry = se
}
}
if subEntry == nil {
addEntry(request, state, requiredEntry, pathname)
} else {
compareEntries(request, state, subEntry, requiredEntry, pathname,
filter)
}
// If a directory: descend (possibly with the directory for the sub).
requiredInode := requiredEntry.Inode()
if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
var subInode *filesystem.DirectoryInode
if subEntry != nil {
if si, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
subInode = si
}
}
compareDirectories(request, state, subInode, requiredInode,
pathname, filter)
}
}
}
func addDirectory(dirent, oldDirent *filesystem.DirectoryEntry,
fileSystem, oldFS *FileSystem,
directoryPathName string, stat *syscall.Stat_t) error {
myPathName := path.Join(directoryPathName, dirent.Name)
if stat.Ino == fileSystem.inodeNumber {
return errors.New("Recursive directory: " + myPathName)
}
if _, ok := fileSystem.InodeTable[stat.Ino]; ok {
return errors.New("Hardlinked directory: " + myPathName)
}
inode := new(filesystem.DirectoryInode)
dirent.SetInode(inode)
fileSystem.InodeTable[stat.Ino] = inode
inode.Mode = filesystem.FileMode(stat.Mode)
inode.Uid = stat.Uid
inode.Gid = stat.Gid
var oldInode *filesystem.DirectoryInode
if oldDirent != nil {
if oi, ok := oldDirent.Inode().(*filesystem.DirectoryInode); ok {
oldInode = oi
}
}
err, copied := scanDirectory(inode, oldInode, fileSystem, oldFS, myPathName)
if err != nil {
return err
}
if copied && filesystem.CompareDirectoriesMetadata(inode, oldInode, nil) {
dirent.SetInode(oldInode)
fileSystem.InodeTable[stat.Ino] = oldInode
}
fileSystem.DirectoryCount++
return nil
}
func addSpecialFile(dirent *filesystem.DirectoryEntry,
fileSystem, oldFS *FileSystem, stat *syscall.Stat_t) error {
if inode, ok := fileSystem.InodeTable[stat.Ino]; ok {
if inode, ok := inode.(*filesystem.SpecialInode); ok {
dirent.SetInode(inode)
return nil
}
return errors.New("Inode changed type: " + dirent.Name)
}
inode := makeSpecialInode(stat)
if oldFS != nil && oldFS.InodeTable != nil {
if oldInode, found := oldFS.InodeTable[stat.Ino]; found {
if oldInode, ok := oldInode.(*filesystem.SpecialInode); ok {
if filesystem.CompareSpecialInodes(inode, oldInode, nil) {
inode = oldInode
}
}
}
}
dirent.SetInode(inode)
fileSystem.InodeTable[stat.Ino] = inode
return nil
}
func (sub *Sub) addInode(request *subproto.UpdateRequest,
requiredEntry *filesystem.DirectoryEntry, myPathName string) {
requiredInode := requiredEntry.Inode()
if name, ok := sub.inodesCreated[requiredEntry.InodeNumber]; ok {
makeHardlink(request, myPathName, name)
return
}
// Try to find a sibling inode.
names := sub.requiredFS.InodeToFilenamesTable()[requiredEntry.InodeNumber]
subFS := sub.FileSystem
if len(names) > 1 {
var sameDataInode filesystem.GenericInode
var sameDataName string
for _, name := range names {
if inum, found := subFS.FilenameToInodeTable()[name]; found {
subInode := sub.FileSystem.InodeTable[inum]
_, sameMetadata, sameData := filesystem.CompareInodes(
subInode, requiredInode, nil)
if sameMetadata && sameData {
makeHardlink(request, myPathName, name)
return
}
if sameData {
sameDataInode = subInode
sameDataName = name
}
}
}
if sameDataInode != nil {
sub.updateMetadata(request, requiredEntry, sameDataName)
makeHardlink(request, myPathName, sameDataName)
return
}
}
if inode, ok := requiredEntry.Inode().(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if _, ok := sub.subObjectCacheUsage[inode.Hash]; ok {
sub.subObjectCacheUsage[inode.Hash]++
} else {
// Not in object cache: grab it from file-system.
request.FilesToCopyToCache = append(
request.FilesToCopyToCache,
sub.getFileToCopy(myPathName, inode.Hash))
sub.subObjectCacheUsage[inode.Hash] = 1
}
}
}
var inode subproto.Inode
inode.Name = myPathName
inode.GenericInode = requiredEntry.Inode()
request.InodesToMake = append(request.InodesToMake, inode)
sub.inodesCreated[requiredEntry.InodeNumber] = myPathName
}
func scanDirectory(directory, oldDirectory *filesystem.DirectoryInode,
fileSystem, oldFS *FileSystem, myPathName string) (error, bool) {
file, err := os.Open(path.Join(fileSystem.rootDirectoryName, myPathName))
if err != nil {
return err, false
}
names, err := file.Readdirnames(-1)
file.Close()
if err != nil {
return err, false
}
sort.Strings(names)
entryList := make([]*filesystem.DirectoryEntry, 0, len(names))
var copiedDirents int
for _, name := range names {
if directory == &fileSystem.DirectoryInode && name == ".subd" {
continue
}
filename := path.Join(myPathName, name)
if fileSystem.configuration.ScanFilter.Match(filename) {
continue
}
var stat syscall.Stat_t
err := syscall.Lstat(path.Join(fileSystem.rootDirectoryName, filename),
&stat)
if err != nil {
if err == syscall.ENOENT {
continue
}
return err, false
}
if stat.Dev != fileSystem.dev {
continue
}
if checkScanDisableRequest() {
return errors.New("DisableScan"), false
}
myGC()
dirent := new(filesystem.DirectoryEntry)
dirent.Name = name
dirent.InodeNumber = stat.Ino
var oldDirent *filesystem.DirectoryEntry
if oldDirectory != nil {
index := len(entryList)
if len(oldDirectory.EntryList) > index &&
oldDirectory.EntryList[index].Name == name {
oldDirent = oldDirectory.EntryList[index]
}
}
if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
err = addDirectory(dirent, oldDirent, fileSystem, oldFS, myPathName,
&stat)
} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
err = addRegularFile(dirent, fileSystem, oldFS, myPathName, &stat)
} else if stat.Mode&syscall.S_IFMT == syscall.S_IFLNK {
err = addSymlink(dirent, fileSystem, oldFS, myPathName, &stat)
} else if stat.Mode&syscall.S_IFMT == syscall.S_IFSOCK {
continue
} else {
err = addSpecialFile(dirent, fileSystem, oldFS, &stat)
}
if err != nil {
if err == syscall.ENOENT {
continue
}
return err, false
}
if oldDirent != nil && *dirent == *oldDirent {
dirent = oldDirent
copiedDirents++
}
entryList = append(entryList, dirent)
}
if oldDirectory != nil && len(entryList) == copiedDirents &&
len(entryList) == len(oldDirectory.EntryList) {
directory.EntryList = oldDirectory.EntryList
return nil, true
} else {
directory.EntryList = entryList
return nil, false
}
}
// Returns true if there is a failure due to missing computed files.
func (sub *Sub) compareDirectories(request *subproto.UpdateRequest,
subDirectory, requiredDirectory *filesystem.DirectoryInode,
myPathName string, deleteMissingComputedFiles bool,
logger *log.Logger) bool {
// First look for entries that should be deleted.
if sub.filter != nil && subDirectory != nil {
for name := range subDirectory.EntriesByName {
pathname := path.Join(myPathName, name)
if sub.filter.Match(pathname) {
continue
}
if _, ok := requiredDirectory.EntriesByName[name]; !ok {
request.PathsToDelete = append(request.PathsToDelete, pathname)
}
}
}
for name, requiredEntry := range requiredDirectory.EntriesByName {
pathname := path.Join(myPathName, name)
if sub.filter != nil && sub.filter.Match(pathname) {
continue
}
var subEntry *filesystem.DirectoryEntry
if subDirectory != nil {
if se, ok := subDirectory.EntriesByName[name]; ok {
subEntry = se
}
}
requiredInode := requiredEntry.Inode()
if _, ok := requiredInode.(*filesystem.ComputedRegularInode); ok {
// Replace with computed file.
inode, ok := sub.ComputedInodes[pathname]
if !ok {
if deleteMissingComputedFiles {
if subEntry != nil {
request.PathsToDelete = append(request.PathsToDelete,
pathname)
}
continue
}
logger.Printf(
"compareDirectories(%s): missing computed file: %s\n",
sub, pathname)
return true
}
setComputedFileMtime(inode, subEntry)
newEntry := new(filesystem.DirectoryEntry)
newEntry.Name = name
newEntry.InodeNumber = requiredEntry.InodeNumber
newEntry.SetInode(inode)
requiredEntry = newEntry
}
if subEntry == nil {
sub.addEntry(request, requiredEntry, pathname)
} else {
sub.compareEntries(request, subEntry, requiredEntry, pathname)
}
// If a directory: descend (possibly with the directory for the sub).
if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
var subInode *filesystem.DirectoryInode
if subEntry != nil {
if si, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
subInode = si
}
}
sub.compareDirectories(request, subInode, requiredInode, pathname,
deleteMissingComputedFiles, logger)
}
}
return false
}
func addInode(request *subproto.UpdateRequest, state *state,
requiredEntry *filesystem.DirectoryEntry, myPathName string) {
requiredInode := requiredEntry.Inode()
if name, ok := state.inodesCreated[requiredEntry.InodeNumber]; ok {
makeHardlink(request, myPathName, name)
return
}
// Try to find a sibling inode.
names := state.requiredFS.InodeToFilenamesTable[requiredEntry.InodeNumber]
if len(names) > 1 {
var sameDataInode filesystem.GenericInode
var sameDataName string
for _, name := range names {
if inum, found := state.getSubInodeFromFilename(name); found {
subInode := state.subFS.InodeTable[inum]
_, sameMetadata, sameData := filesystem.CompareInodes(
subInode, requiredInode, nil)
if sameMetadata && sameData {
makeHardlink(request, myPathName, name)
return
}
if sameData {
sameDataInode = subInode
sameDataName = name
}
}
}
if sameDataInode != nil {
updateMetadata(request, state, requiredEntry, sameDataName)
makeHardlink(request, myPathName, sameDataName)
return
}
}
if inode, ok := requiredEntry.Inode().(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if _, ok := state.subObjectCacheUsage[inode.Hash]; ok {
state.subObjectCacheUsage[inode.Hash]++
} else {
// Not in object cache: grab it from file-system.
if state.subFS.HashToInodesTable == nil {
state.subFS.BuildHashToInodesTable()
}
if ilist, ok := state.subFS.HashToInodesTable[inode.Hash]; ok {
var fileToCopy subproto.FileToCopyToCache
fileToCopy.Name =
state.subFS.InodeToFilenamesTable[ilist[0]][0]
fileToCopy.Hash = inode.Hash
request.FilesToCopyToCache = append(
request.FilesToCopyToCache, fileToCopy)
state.subObjectCacheUsage[inode.Hash] = 1
} else {
panic("No object in cache for: " + myPathName)
}
}
}
}
var inode subproto.Inode
inode.Name = myPathName
inode.GenericInode = requiredEntry.Inode()
request.InodesToMake = append(request.InodesToMake, inode)
state.inodesCreated[requiredEntry.InodeNumber] = myPathName
}