func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 {
return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
}
if !tf.IsDeleted() {
if debugDB {
l.Debugf("mark deleted; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
}
ts := clock(tf.LocalVersion)
f := protocol.FileInfo{
Name: tf.Name,
Version: lamport.Default.Tick(tf.Version),
LocalVersion: ts,
Flags: tf.Flags | protocol.FlagDeleted,
Modified: tf.Modified,
}
bs, _ := f.MarshalXDR()
if debugDB {
l.Debugf("batch.Put %p %x", batch, dbi.Key())
}
batch.Put(dbi.Key(), bs)
ldbUpdateGlobal(db, batch, folder, device, deviceKeyName(dbi.Key()), f.Version)
return ts
}
return 0
})
}
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
if truncate {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(bs)
return tf, err
} else {
var tf protocol.FileInfo
err := tf.UnmarshalXDR(bs)
return tf, err
}
}
func ldbGetGlobal(db *leveldb.DB, folder, file []byte) protocol.FileInfo {
k := globalKey(folder, file)
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
if debugDB {
l.Debugf("created snapshot %p", snap)
}
defer func() {
if debugDB {
l.Debugf("close snapshot %p", snap)
}
snap.Release()
}()
if debugDB {
l.Debugf("snap.Get %p %x", snap, k)
}
bs, err := snap.Get(k, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}
}
if err != nil {
panic(err)
}
var vl versionList
err = vl.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
l.Debugln(k)
panic("no versions?")
}
k = deviceKey(folder, vl.versions[0].device, file)
if debugDB {
l.Debugf("snap.Get %p %x", snap, k)
}
bs, err = snap.Get(k, nil)
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f
}
// Test that updating a file removes it's old blocks from the blockmap
func TestCopierCleanup(t *testing.T) {
iterFn := func(folder, file string, index uint32) bool {
return true
}
fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
db, _ := leveldb.Open(storage.NewMemStorage(), nil)
m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
m.AddFolder(fcfg)
// Create a file
file := protocol.FileInfo{
Name: "test",
Flags: 0,
Modified: 0,
Blocks: []protocol.BlockInfo{blocks[0]},
}
// Add file to index
m.updateLocal("default", file)
if !m.finder.Iterate(blocks[0].Hash, iterFn) {
t.Error("Expected block not found")
}
file.Blocks = []protocol.BlockInfo{blocks[1]}
file.Version++
// Update index (removing old blocks)
m.updateLocal("default", file)
if m.finder.Iterate(blocks[0].Hash, iterFn) {
t.Error("Unexpected block found")
}
if !m.finder.Iterate(blocks[1].Hash, iterFn) {
t.Error("Expected block not found")
}
file.Blocks = []protocol.BlockInfo{blocks[0]}
file.Version++
// Update index (removing old blocks)
m.updateLocal("default", file)
if !m.finder.Iterate(blocks[0].Hash, iterFn) {
t.Error("Unexpected block found")
}
if m.finder.Iterate(blocks[1].Hash, iterFn) {
t.Error("Expected block not found")
}
}
func (m *Model) updateLocal(folder string, f protocol.FileInfo) {
f.LocalVersion = 0
m.fmut.RLock()
m.folderFiles[folder].Update(protocol.LocalDeviceID, []protocol.FileInfo{f})
m.fmut.RUnlock()
events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
"folder": folder,
"name": f.Name,
"modified": time.Unix(f.Modified, 0),
"flags": fmt.Sprintf("0%o", f.Flags),
"size": f.Size(),
})
}
func ldbInsert(batch dbWriter, folder, device, name []byte, file protocol.FileInfo) uint64 {
if debug {
l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file)
}
if file.LocalVersion == 0 {
file.LocalVersion = clock(0)
}
nk := deviceKey(folder, device, name)
batch.Put(nk, file.MarshalXDR())
return file.LocalVersion
}
func ldbGet(db *leveldb.DB, folder, device, file []byte) protocol.FileInfo {
nk := deviceKey(folder, device, file)
bs, err := db.Get(nk, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}
}
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f
}
// handleFile queues the copies and pulls as necessary for a single new or
// changed file.
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
// We are supposed to copy the entire file, and then fetch nothing. We
// are only updating metadata, so we don't actually *need* to make the
// copy.
if debug {
l.Debugln(p, "taking shortcut on", file.Name)
}
if file.IsSymlink() {
p.shortcutSymlink(curFile, file)
} else {
p.shortcutFile(file)
}
return
}
scanner.PopulateOffsets(file.Blocks)
// Figure out the absolute filenames we need once and for all
tempName := filepath.Join(p.dir, defTempNamer.TempName(file.Name))
realName := filepath.Join(p.dir, file.Name)
reused := 0
var blocks []protocol.BlockInfo
// Check for an old temporary file which might have some blocks we could
// reuse.
tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize)
if err == nil {
// Check for any reusable blocks in the temp file
tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
// block.String() returns a string unique to the block
existingBlocks := make(map[string]bool, len(tempCopyBlocks))
for _, block := range tempCopyBlocks {
existingBlocks[block.String()] = true
}
// Since the blocks are already there, we don't need to get them.
for _, block := range file.Blocks {
_, ok := existingBlocks[block.String()]
if !ok {
blocks = append(blocks, block)
}
}
// The sharedpullerstate will know which flags to use when opening the
// temp file depending if we are reusing any blocks or not.
reused = len(file.Blocks) - len(blocks)
if reused == 0 {
// Otherwise, discard the file ourselves in order for the
// sharedpuller not to panic when it fails to exlusively create a
// file which already exists
os.Remove(tempName)
}
} else {
blocks = file.Blocks
}
s := sharedPullerState{
file: file,
folder: p.folder,
tempName: tempName,
realName: realName,
copyTotal: uint32(len(blocks)),
copyNeeded: uint32(len(blocks)),
reused: uint32(reused),
}
if debug {
l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused)
}
cs := copyBlocksState{
sharedPullerState: &s,
blocks: blocks,
}
copyChan <- cs
}
// On the 10th iteration, we start hashing the content which we receive by
// following blockfinder's instructions. Make sure that the copier routine
// hashes the content when asked, and pulls if it fails to find the block.
func TestLastResortPulling(t *testing.T) {
fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
db, _ := leveldb.Open(storage.NewMemStorage(), nil)
m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
m.AddFolder(fcfg)
// Add a file to index (with the incorrect block representation, as content
// doesn't actually match the block list)
file := protocol.FileInfo{
Name: "empty",
Flags: 0,
Modified: 0,
Blocks: []protocol.BlockInfo{blocks[0]},
}
m.updateLocal("default", file)
// Pretend that we are handling a new file of the same content but
// with a different name (causing to copy that particular block)
file.Name = "newfile"
iterFn := func(folder, file string, index uint32) bool {
return true
}
// Check that that particular block is there
if !m.finder.Iterate(blocks[0].Hash, iterFn) {
t.Error("Expected block not found")
}
p := Puller{
folder: "default",
dir: "testdata",
model: m,
}
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, 1)
finisherChan := make(chan *sharedPullerState, 1)
// Run a single copier routine with checksumming enabled
go p.copierRoutine(copyChan, pullChan, finisherChan, true)
p.handleFile(file, copyChan, finisherChan)
// Copier should hash empty file, realise that the region it has read
// doesn't match the hash which was advertised by the block map, fix it
// and ask to pull the block.
<-pullChan
// Verify that it did fix the incorrect hash.
if m.finder.Iterate(blocks[0].Hash, iterFn) {
t.Error("Found unexpected block")
}
if !m.finder.Iterate(scanner.SHA256OfNothing, iterFn) {
t.Error("Expected block not found")
}
(<-finisherChan).fd.Close()
os.Remove(filepath.Join("testdata", defTempNamer.TempName("newfile")))
}