func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
debug.Log("called")
sn.Lock()
defer sn.Unlock()
for id := range sn.repo.List(restic.SnapshotFile, ctx.Done()) {
if sn.processed.Has(id) {
debug.Log("skipping snapshot %v, already in list", id.Str())
continue
}
debug.Log("found snapshot id %v", id.Str())
snapshot, err := restic.LoadSnapshot(sn.repo, id)
if err != nil {
return err
}
timestamp := snapshot.Time.Format(time.RFC3339)
for i := 1; ; i++ {
if _, ok := sn.knownSnapshots[timestamp]; !ok {
break
}
timestamp = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
}
debug.Log(" add %v as dir %v", id.Str(), timestamp)
sn.knownSnapshots[timestamp] = SnapshotWithId{snapshot, id}
sn.processed.Insert(id)
}
return nil
}
func runLs(gopts GlobalOptions, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errors.Fatalf("no snapshot ID given")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
id, err := restic.FindSnapshot(repo, args[0])
if err != nil {
return err
}
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
return printTree("", repo, *sn.Tree)
}
func findLatestSnapshot(repo *repository.Repository, targets []string) (backend.ID, error) {
var (
latest time.Time
latestID backend.ID
found bool
)
for snapshotID := range repo.List(backend.Snapshot, make(chan struct{})) {
snapshot, err := restic.LoadSnapshot(repo, snapshotID)
if err != nil {
return backend.ID{}, fmt.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && samePaths(snapshot.Paths, targets) {
latest = snapshot.Time
latestID = snapshotID
found = true
}
}
if !found {
return backend.ID{}, errNoSnapshotFound
}
return latestID, nil
}
func (cmd CmdLs) Execute(args []string) error {
if len(args) < 1 || len(args) > 2 {
return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
}
repo, err := cmd.global.OpenRepository()
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
id, err := restic.FindSnapshot(repo, args[0])
if err != nil {
return err
}
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
cmd.global.Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
return cmd.printTree("", repo, *sn.Tree)
}
func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs) {
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
go mount(t, global, mountpoint)
waitForMount(t, mountpoint)
defer umount(t, global, mountpoint)
if !snapshotsDirExists(t, mountpoint) {
t.Fatal(`virtual directory "snapshots" doesn't exist`)
}
ids := listSnapshots(t, repodir)
t.Logf("found %v snapshots in repo: %v", len(ids), ids)
namesInSnapshots := listSnapshots(t, mountpoint)
t.Logf("found %v snapshots in fuse mount: %v", len(namesInSnapshots), namesInSnapshots)
Assert(t,
len(namesInSnapshots) == len(snapshotIDs),
"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))
namesMap := make(map[string]bool)
for _, name := range namesInSnapshots {
namesMap[name] = false
}
for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(repo, id)
OK(t, err)
ts := snapshot.Time.Format(time.RFC3339)
present, ok := namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}
for i := 1; present; i++ {
ts = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
present, ok = namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}
if !present {
break
}
}
namesMap[ts] = true
}
for name, present := range namesMap {
Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
}
}
func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
sn.Lock()
defer sn.Unlock()
for id := range sn.repo.List(backend.Snapshot, ctx.Done()) {
snapshot, err := restic.LoadSnapshot(sn.repo, id)
if err != nil {
return err
}
sn.knownSnapshots[snapshot.Time.Format(time.RFC3339)] = SnapshotWithId{snapshot, id}
}
return nil
}
func loadTreeFromSnapshot(repo *repository.Repository, id backend.ID) (backend.ID, error) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
debug.Log("Checker.loadTreeFromSnapshot", "error loading snapshot %v: %v", id.Str(), err)
return backend.ID{}, err
}
if sn.Tree == nil {
debug.Log("Checker.loadTreeFromSnapshot", "snapshot %v has no tree", id.Str())
return backend.ID{}, fmt.Errorf("snapshot %v has no tree", id)
}
return *sn.Tree, nil
}
func loadTreeFromSnapshot(repo restic.Repository, id restic.ID) (restic.ID, error) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
debug.Log("error loading snapshot %v: %v", id.Str(), err)
return restic.ID{}, err
}
if sn.Tree == nil {
debug.Log("snapshot %v has no tree", id.Str())
return restic.ID{}, errors.Errorf("snapshot %v has no tree", id)
}
return *sn.Tree, nil
}
func printSnapshots(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
for id := range repo.List(backend.Snapshot, done) {
snapshot, err := restic.LoadSnapshot(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
continue
}
fmt.Fprintf(wr, "snapshot_id: %v\n", id)
err = prettyPrintJSON(wr, snapshot)
if err != nil {
return err
}
}
return nil
}
func runLs(gopts GlobalOptions, args []string) error {
if len(args) < 1 || len(args) > 2 {
return errors.Fatalf("no snapshot ID given")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
snapshotIDString := args[0]
var id restic.ID
if snapshotIDString == "latest" {
id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
if err != nil {
Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
}
} else {
id, err = restic.FindSnapshot(repo, snapshotIDString)
if err != nil {
Exitf(1, "invalid id %q: %v", snapshotIDString, err)
}
}
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
return printTree("", repo, *sn.Tree)
}
func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error {
debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest)
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
results, err := c.findInTree(repo, *sn.Tree, "")
if err != nil {
return err
}
if len(results) == 0 {
return nil
}
c.global.Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
for _, res := range results {
res.node.Name = filepath.Join(res.path, res.node.Name)
c.global.Printf(" %s\n", res.node)
}
return nil
}
func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
return err
}
results, err := findInTree(repo, pat, *sn.Tree, "")
if err != nil {
return err
}
if len(results) == 0 {
return nil
}
Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
for _, res := range results {
res.node.Name = filepath.Join(res.path, res.node.Name)
Printf(" %s\n", res.node)
}
return nil
}
func (cmd CmdSnapshots) Execute(args []string) error {
if len(args) != 0 {
return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage())
}
repo, err := cmd.global.OpenRepository()
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
tab := NewTable()
tab.Header = fmt.Sprintf("%-8s %-19s %-10s %s", "ID", "Date", "Source", "Directory")
tab.RowFormat = "%-8s %-19s %-10s %s"
done := make(chan struct{})
defer close(done)
list := []*restic.Snapshot{}
for id := range repo.List(backend.Snapshot, done) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
continue
}
pos := sort.Search(len(list), func(i int) bool {
return list[i].Time.After(sn.Time)
})
if pos < len(list) {
list = append(list, nil)
copy(list[pos+1:], list[pos:])
list[pos] = sn
} else {
list = append(list, sn)
}
}
plen, err := repo.PrefixLength(backend.Snapshot)
if err != nil {
return err
}
for _, sn := range list {
if len(sn.Paths) == 0 {
continue
}
id := sn.ID()
tab.Rows = append(tab.Rows, []interface{}{hex.EncodeToString(id[:plen/2]), sn.Time.Format(TimeFormat), sn.Hostname, sn.Paths[0]})
if len(sn.Paths) > 1 {
for _, path := range sn.Paths[1:] {
tab.Rows = append(tab.Rows, []interface{}{"", "", "", path})
}
}
}
tab.Write(os.Stdout)
return nil
}
func TestMount(t *testing.T) {
if !RunFuseTest {
t.Skip("Skipping fuse tests")
}
checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) {
snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots"))
OK(t, err)
namesInSnapshots, err := snapshotsDir.Readdirnames(-1)
OK(t, err)
Assert(t,
len(namesInSnapshots) == len(snapshotIDs),
"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))
namesMap := make(map[string]bool)
for _, name := range namesInSnapshots {
namesMap[name] = false
}
for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(repo, id)
OK(t, err)
_, ok := namesMap[snapshot.Time.Format(time.RFC3339)]
Assert(t, ok, "Snapshot %s isn't present in fuse dir", snapshot.Time.Format(time.RFC3339))
namesMap[snapshot.Time.Format(time.RFC3339)] = true
}
for name, present := range namesMap {
Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
}
OK(t, snapshotsDir.Close())
}
withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
cmdInit(t, global)
repo, err := global.OpenRepository()
OK(t, err)
mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
OK(t, err)
// We remove the mountpoint now to check that cmdMount creates it
RemoveAll(t, mountpoint)
ready := make(chan struct{}, 2)
done := make(chan struct{})
go cmdMount(t, global, mountpoint, ready, done)
<-ready
defer close(done)
OK(t, waitForMount(mountpoint))
mountpointDir, err := os.Open(mountpoint)
OK(t, err)
names, err := mountpointDir.Readdirnames(-1)
OK(t, err)
Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`)
OK(t, mountpointDir.Close())
checkSnapshots(repo, mountpoint, []backend.ID{})
datafile := filepath.Join("testdata", "backup-data.tar.gz")
fd, err := os.Open(datafile)
if os.IsNotExist(err) {
t.Skipf("unable to find data file %q, skipping", datafile)
return
}
OK(t, err)
OK(t, fd.Close())
SetupTarTestFixture(t, env.testdata, datafile)
// first backup
cmdBackup(t, global, []string{env.testdata}, nil)
snapshotIDs := cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 1,
"expected one snapshot, got %v", snapshotIDs)
checkSnapshots(repo, mountpoint, snapshotIDs)
// second backup, implicit incremental
cmdBackup(t, global, []string{env.testdata}, nil)
snapshotIDs = cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 2,
"expected two snapshots, got %v", snapshotIDs)
checkSnapshots(repo, mountpoint, snapshotIDs)
// third backup, explicit incremental
cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
snapshotIDs = cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 3,
"expected three snapshots, got %v", snapshotIDs)
checkSnapshots(repo, mountpoint, snapshotIDs)
})
}
// Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
// used to compare the files to the ones archived at the time this snapshot was
// taken.
func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
paths = unique(paths)
sort.Sort(baseNameSlice(paths))
debug.Log("start for %v", paths)
debug.RunHook("Archiver.Snapshot", nil)
// signal the whole pipeline to stop
done := make(chan struct{})
var err error
p.Start()
defer p.Done()
// create new snapshot
sn, err := restic.NewSnapshot(paths, tags)
if err != nil {
return nil, restic.ID{}, err
}
sn.Excludes = arch.Excludes
jobs := archivePipe{}
// use parent snapshot (if some was given)
if parentID != nil {
sn.Parent = parentID
// load parent snapshot
parent, err := restic.LoadSnapshot(arch.repo, *parentID)
if err != nil {
return nil, restic.ID{}, err
}
// start walker on old tree
ch := make(chan walk.TreeJob)
go walk.Tree(arch.repo, *parent.Tree, done, ch)
jobs.Old = ch
} else {
// use closed channel
ch := make(chan walk.TreeJob)
close(ch)
jobs.Old = ch
}
// start walker
pipeCh := make(chan pipe.Job)
resCh := make(chan pipe.Result, 1)
go func() {
pipe.Walk(paths, arch.SelectFilter, done, pipeCh, resCh)
debug.Log("pipe.Walk done")
}()
jobs.New = pipeCh
ch := make(chan pipe.Job)
go jobs.compare(done, ch)
var wg sync.WaitGroup
entCh := make(chan pipe.Entry)
dirCh := make(chan pipe.Dir)
// split
wg.Add(1)
go func() {
pipe.Split(ch, dirCh, entCh)
debug.Log("split done")
close(dirCh)
close(entCh)
wg.Done()
}()
// run workers
for i := 0; i < maxConcurrency; i++ {
wg.Add(2)
go arch.fileWorker(&wg, p, done, entCh)
go arch.dirWorker(&wg, p, done, dirCh)
}
// run index saver
var wgIndexSaver sync.WaitGroup
stopIndexSaver := make(chan struct{})
wgIndexSaver.Add(1)
go arch.saveIndexes(&wgIndexSaver, stopIndexSaver)
// wait for all workers to terminate
debug.Log("wait for workers")
wg.Wait()
// stop index saver
close(stopIndexSaver)
wgIndexSaver.Wait()
debug.Log("workers terminated")
// receive the top-level tree
root := (<-resCh).(*restic.Node)
debug.Log("root node received: %v", root.Subtree.Str())
sn.Tree = root.Subtree
// save snapshot
id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
if err != nil {
return nil, restic.ID{}, err
}
debug.Log("saved snapshot %v", id.Str())
// flush repository
err = arch.repo.Flush()
if err != nil {
return nil, restic.ID{}, err
}
// save index
err = arch.repo.SaveIndex()
if err != nil {
debug.Log("error saving index: %v", err)
return nil, restic.ID{}, err
}
debug.Log("saved indexes")
return sn, id, nil
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatalf("wrong number of arguments")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
tab := NewTable()
tab.Header = fmt.Sprintf("%-8s %-19s %-10s %-10s %s", "ID", "Date", "Host", "Tags", "Directory")
tab.RowFormat = "%-8s %-19s %-10s %-10s %s"
done := make(chan struct{})
defer close(done)
list := []*restic.Snapshot{}
for id := range repo.List(restic.SnapshotFile, done) {
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
continue
}
if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
pos := sort.Search(len(list), func(i int) bool {
return list[i].Time.After(sn.Time)
})
if pos < len(list) {
list = append(list, nil)
copy(list[pos+1:], list[pos:])
list[pos] = sn
} else {
list = append(list, sn)
}
}
}
for _, sn := range list {
if len(sn.Paths) == 0 {
continue
}
firstTag := ""
if len(sn.Tags) > 0 {
firstTag = sn.Tags[0]
}
tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
rows := len(sn.Paths)
if len(sn.Tags) > rows {
rows = len(sn.Tags)
}
for i := 1; i < rows; i++ {
path := ""
if len(sn.Paths) > i {
path = sn.Paths[i]
}
tag := ""
if len(sn.Tags) > i {
tag = sn.Tags[i]
}
tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
}
}
tab.Write(os.Stdout)
return nil
}