func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { if truncate { var tf FileInfoTruncated err := tf.UnmarshalXDR(bs) return tf, err } var tf protocol.FileInfo err := tf.UnmarshalXDR(bs) return tf, err }
// shortcutSymlink changes the symlinks type if necessary. func (p *rwFolder) shortcutSymlink(file protocol.FileInfo) (err error) { tt := symlinks.TargetFile if file.IsDirectory() { tt = symlinks.TargetDirectory } err = symlinks.ChangeType(filepath.Join(p.dir, file.Name), tt) if err != nil { l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err) p.newError(file.Name, err) } return }
func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.FileInfo) int64 { l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file) if file.LocalVersion == 0 { file.LocalVersion = clock(0) } name := []byte(file.Name) nk := t.db.deviceKey(folder, device, name) t.Put(nk, file.MustMarshalXDR()) return file.LocalVersion }
func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) { bs, err := db.Get(key, nil) if err == leveldb.ErrNotFound { return protocol.FileInfo{}, false } if err != nil { panic(err) } var f protocol.FileInfo err = f.UnmarshalXDR(bs) if err != nil { panic(err) } return f, true }
// shortcutFile sets file mode and modification time, when that's the only // thing that has changed. func (p *rwFolder) shortcutFile(file protocol.FileInfo) error { realName := filepath.Join(p.dir, file.Name) if !p.ignorePermissions(file) { if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil { l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", p.folder, file.Name, err) p.newError(file.Name, err) return err } } t := time.Unix(file.Modified, 0) if err := os.Chtimes(realName, t, t); err != nil { // Try using virtual mtimes info, err := os.Stat(realName) if err != nil { l.Infof("Puller (folder %q, file %q): shortcut: unable to stat file: %v", p.folder, file.Name, err) p.newError(file.Name, err) return err } p.virtualMtimeRepo.UpdateMtime(file.Name, info.ModTime(), t) } // This may have been a conflict. We should merge the version vectors so // that our clock doesn't move backwards. if cur, ok := p.model.CurrentFolderFile(p.folder, file.Name); ok { file.Version = file.Version.Merge(cur.Version) } return nil }
func dump(ldb *leveldb.DB) { it := ldb.NewIterator(nil, nil) var dev protocol.DeviceID for it.Next() { key := it.Key() switch key[0] { case db.KeyTypeDevice: folder := nulString(key[1 : 1+64]) devBytes := key[1+64 : 1+64+32] name := nulString(key[1+64+32:]) copy(dev[:], devBytes) fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev) var f protocol.FileInfo err := f.UnmarshalXDR(it.Value()) if err != nil { log.Fatal(err) } fmt.Printf(" N:%q\n F:%#o\n M:%d\n V:%v\n S:%d\n B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks)) case db.KeyTypeGlobal: folder := nulString(key[1 : 1+64]) name := nulString(key[1+64:]) fmt.Printf("[global] F:%q N:%q V:%x\n", folder, name, it.Value()) case db.KeyTypeBlock: folder := nulString(key[1 : 1+64]) hash := key[1+64 : 1+64+32] name := nulString(key[1+64+32:]) fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value())) case db.KeyTypeDeviceStatistic: fmt.Printf("[dstat]\n %x\n %x\n", it.Key(), it.Value()) case db.KeyTypeFolderStatistic: fmt.Printf("[fstat]\n %x\n %x\n", it.Key(), it.Value()) case db.KeyTypeVirtualMtime: fmt.Printf("[mtime]\n %x\n %x\n", it.Key(), it.Value()) default: fmt.Printf("[???]\n %x\n %x\n", it.Key(), it.Value()) } } }
func SymlinkTypeEqual(disk symlinks.TargetType, f protocol.FileInfo) bool { // If the target is missing, Unix never knows what type of symlink it is // and Windows always knows even if there is no target. Which means that // without this special check a Unix node would be fighting with a Windows // node about whether or not the target is known. Basically, if you don't // know and someone else knows, just accept it. The fact that you don't // know means you are on Unix, and on Unix you don't really care what the // target type is. The moment you do know, and if something doesn't match, // that will propagate through the cluster. switch disk { case symlinks.TargetUnknown: return true case symlinks.TargetDirectory: return f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0 case symlinks.TargetFile: return !f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0 } panic("unknown symlink TargetType") }
// deleteFile attempts to delete the given file func (p *rwFolder) deleteFile(file protocol.FileInfo) { var err error events.Default.Log(events.ItemStarted, map[string]string{ "folder": p.folder, "item": file.Name, "type": "file", "action": "delete", }) defer func() { events.Default.Log(events.ItemFinished, map[string]interface{}{ "folder": p.folder, "item": file.Name, "error": events.Error(err), "type": "file", "action": "delete", }) }() realName := filepath.Join(p.dir, file.Name) cur, ok := p.model.CurrentFolderFile(p.folder, file.Name) if ok && p.inConflict(cur.Version, file.Version) { // There is a conflict here. Move the file to a conflict copy instead // of deleting. Also merge with the version vector we had, to indicate // we have resolved the conflict. file.Version = file.Version.Merge(cur.Version) err = osutil.InWritableDir(p.moveForConflict, realName) } else if p.versioner != nil { err = osutil.InWritableDir(p.versioner.Archive, realName) } else { err = osutil.InWritableDir(osutil.Remove, realName) } if err == nil || os.IsNotExist(err) { // It was removed or it doesn't exist to start with p.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile} } else if _, serr := os.Lstat(realName); serr != nil && !os.IsPermission(serr) { // We get an error just looking at the file, and it's not a permission // problem. Lets assume the error is in fact some variant of "file // does not exist" (possibly expressed as some parent being a file and // not a directory etc) and that the delete is handled. p.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile} } else { l.Infof("Puller (folder %q, file %q): delete: %v", p.folder, file.Name, err) p.newError(file.Name, err) } }
// handleFile queues the copies and pulls as necessary for a single new or // changed file. func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) { curFile, hasCurFile := p.model.CurrentFolderFile(p.folder, file.Name) if hasCurFile && len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) { // We are supposed to copy the entire file, and then fetch nothing. We // are only updating metadata, so we don't actually *need* to make the // copy. l.Debugln(p, "taking shortcut on", file.Name) events.Default.Log(events.ItemStarted, map[string]string{ "folder": p.folder, "item": file.Name, "type": "file", "action": "metadata", }) p.queue.Done(file.Name) var err error if file.IsSymlink() { err = p.shortcutSymlink(file) } else { err = p.shortcutFile(file) } events.Default.Log(events.ItemFinished, map[string]interface{}{ "folder": p.folder, "item": file.Name, "error": events.Error(err), "type": "file", "action": "metadata", }) if err != nil { l.Infoln("Puller: shortcut:", err) p.newError(file.Name, err) } else { p.dbUpdates <- dbUpdateJob{file, dbUpdateShortcutFile} } return } // Figure out the absolute filenames we need once and for all tempName := filepath.Join(p.dir, defTempNamer.TempName(file.Name)) realName := filepath.Join(p.dir, file.Name) if hasCurFile && !curFile.IsDirectory() && !curFile.IsSymlink() { // Check that the file on disk is what we expect it to be according to // the database. If there's a mismatch here, there might be local // changes that we don't know about yet and we should scan before // touching the file. If we can't stat the file we'll just pull it. if info, err := osutil.Lstat(realName); err == nil { mtime := p.virtualMtimeRepo.GetMtime(file.Name, info.ModTime()) if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size() { l.Debugln("file modified but not rescanned; not pulling:", realName) // Scan() is synchronous (i.e. blocks until the scan is // completed and returns an error), but a scan can't happen // while we're in the puller routine. Request the scan in the // background and it'll be handled when the current pulling // sweep is complete. As we do retries, we'll queue the scan // for this file up to ten times, but the last nine of those // scans will be cheap... go p.Scan([]string{file.Name}) return } } } scanner.PopulateOffsets(file.Blocks) reused := 0 var blocks []protocol.BlockInfo var blocksSize int64 // Check for an old temporary file which might have some blocks we could // reuse. tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize, 0, nil) if err == nil { // Check for any reusable blocks in the temp file tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks) // block.String() returns a string unique to the block existingBlocks := make(map[string]struct{}, len(tempCopyBlocks)) for _, block := range tempCopyBlocks { existingBlocks[block.String()] = struct{}{} } // Since the blocks are already there, we don't need to get them. for _, block := range file.Blocks { _, ok := existingBlocks[block.String()] if !ok { blocks = append(blocks, block) blocksSize += int64(block.Size) } } // The sharedpullerstate will know which flags to use when opening the // temp file depending if we are reusing any blocks or not. reused = len(file.Blocks) - len(blocks) if reused == 0 { // Otherwise, discard the file ourselves in order for the // sharedpuller not to panic when it fails to exclusively create a // file which already exists osutil.InWritableDir(osutil.Remove, tempName) } } else { blocks = file.Blocks blocksSize = file.Size() } if p.checkFreeSpace { if free, err := osutil.DiskFreeBytes(p.dir); err == nil && free < blocksSize { l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, p.folder, p.dir, file.Name, float64(free)/1024/1024, float64(blocksSize)/1024/1024) p.newError(file.Name, errors.New("insufficient space")) return } } events.Default.Log(events.ItemStarted, map[string]string{ "folder": p.folder, "item": file.Name, "type": "file", "action": "update", }) s := sharedPullerState{ file: file, folder: p.folder, tempName: tempName, realName: realName, copyTotal: len(blocks), copyNeeded: len(blocks), reused: reused, ignorePerms: p.ignorePermissions(file), version: curFile.Version, mut: sync.NewMutex(), sparse: p.allowSparse, } l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused) cs := copyBlocksState{ sharedPullerState: &s, blocks: blocks, } copyChan <- cs }
// updateGlobal adds this device+version to the version list for the given // file. If the device is already present in the list, the version is updated. // If the file does not have an entry in the global list, it is created. func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.FileInfo, globalSize *sizeTracker) bool { l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version) name := []byte(file.Name) gk := t.db.globalKey(folder, name) svl, err := t.Get(gk, nil) if err != nil && err != leveldb.ErrNotFound { panic(err) } var fl versionList var oldFile protocol.FileInfo var hasOldFile bool // Remove the device from the current version list if svl != nil { err = fl.UnmarshalXDR(svl) if err != nil { panic(err) } for i := range fl.versions { if bytes.Compare(fl.versions[i].device, device) == 0 { if fl.versions[i].version.Equal(file.Version) { // No need to do anything return false } if i == 0 { // Keep the current newest file around so we can subtract it from // the globalSize if we replace it. oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name) } fl.versions = append(fl.versions[:i], fl.versions[i+1:]...) break } } } nv := fileVersion{ device: device, version: file.Version, } insertedAt := -1 // Find a position in the list to insert this file. The file at the front // of the list is the newer, the "global". for i := range fl.versions { switch fl.versions[i].version.Compare(file.Version) { case protocol.Equal, protocol.Lesser: // The version at this point in the list is equal to or lesser // ("older") than us. We insert ourselves in front of it. fl.versions = insertVersion(fl.versions, i, nv) insertedAt = i goto done case protocol.ConcurrentLesser, protocol.ConcurrentGreater: // The version at this point is in conflict with us. We must pull // the actual file metadata to determine who wins. If we win, we // insert ourselves in front of the loser here. (The "Lesser" and // "Greater" in the condition above is just based on the device // IDs in the version vector, which is not the only thing we use // to determine the winner.) of, ok := t.getFile(folder, fl.versions[i].device, name) if !ok { panic("file referenced in version list does not exist") } if file.WinsConflict(of) { fl.versions = insertVersion(fl.versions, i, nv) insertedAt = i goto done } } } // We didn't find a position for an insert above, so append to the end. fl.versions = append(fl.versions, nv) insertedAt = len(fl.versions) - 1 done: if insertedAt == 0 { // We just inserted a new newest version. Fixup the global size // calculation. if !file.Version.Equal(oldFile.Version) { globalSize.addFile(file) if hasOldFile { // We have the old file that was removed at the head of the list. globalSize.removeFile(oldFile) } else if len(fl.versions) > 1 { // The previous newest version is now at index 1, grab it from there. oldFile, ok := t.getFile(folder, fl.versions[1].device, name) if !ok { panic("file referenced in version list does not exist") } globalSize.removeFile(oldFile) } } } l.Debugf("new global after update: %v", fl) t.Put(gk, fl.MustMarshalXDR()) return true }