Esempio n. 1
0
func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
	debug.Log("called")
	sn.Lock()
	defer sn.Unlock()

	for id := range sn.repo.List(restic.SnapshotFile, ctx.Done()) {
		if sn.processed.Has(id) {
			debug.Log("skipping snapshot %v, already in list", id.Str())
			continue
		}

		debug.Log("found snapshot id %v", id.Str())
		snapshot, err := restic.LoadSnapshot(sn.repo, id)
		if err != nil {
			return err
		}

		timestamp := snapshot.Time.Format(time.RFC3339)
		for i := 1; ; i++ {
			if _, ok := sn.knownSnapshots[timestamp]; !ok {
				break
			}

			timestamp = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
		}

		debug.Log("  add %v as dir %v", id.Str(), timestamp)
		sn.knownSnapshots[timestamp] = SnapshotWithId{snapshot, id}
		sn.processed.Insert(id)
	}
	return nil
}
Esempio n. 2
0
func runLs(gopts GlobalOptions, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errors.Fatalf("no snapshot ID given")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	id, err := restic.FindSnapshot(repo, args[0])
	if err != nil {
		return err
	}

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)

	return printTree("", repo, *sn.Tree)
}
Esempio n. 3
0
func findLatestSnapshot(repo *repository.Repository, targets []string) (backend.ID, error) {
	var (
		latest   time.Time
		latestID backend.ID
		found    bool
	)

	for snapshotID := range repo.List(backend.Snapshot, make(chan struct{})) {
		snapshot, err := restic.LoadSnapshot(repo, snapshotID)
		if err != nil {
			return backend.ID{}, fmt.Errorf("Error listing snapshot: %v", err)
		}
		if snapshot.Time.After(latest) && samePaths(snapshot.Paths, targets) {
			latest = snapshot.Time
			latestID = snapshotID
			found = true
		}
	}

	if !found {
		return backend.ID{}, errNoSnapshotFound
	}

	return latestID, nil
}
Esempio n. 4
0
func (cmd CmdLs) Execute(args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	id, err := restic.FindSnapshot(repo, args[0])
	if err != nil {
		return err
	}

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	cmd.global.Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)

	return cmd.printTree("", repo, *sn.Tree)
}
Esempio n. 5
0
func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs) {
	t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
	go mount(t, global, mountpoint)
	waitForMount(t, mountpoint)
	defer umount(t, global, mountpoint)

	if !snapshotsDirExists(t, mountpoint) {
		t.Fatal(`virtual directory "snapshots" doesn't exist`)
	}

	ids := listSnapshots(t, repodir)
	t.Logf("found %v snapshots in repo: %v", len(ids), ids)

	namesInSnapshots := listSnapshots(t, mountpoint)
	t.Logf("found %v snapshots in fuse mount: %v", len(namesInSnapshots), namesInSnapshots)
	Assert(t,
		len(namesInSnapshots) == len(snapshotIDs),
		"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))

	namesMap := make(map[string]bool)
	for _, name := range namesInSnapshots {
		namesMap[name] = false
	}

	for _, id := range snapshotIDs {
		snapshot, err := restic.LoadSnapshot(repo, id)
		OK(t, err)

		ts := snapshot.Time.Format(time.RFC3339)
		present, ok := namesMap[ts]
		if !ok {
			t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
		}

		for i := 1; present; i++ {
			ts = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
			present, ok = namesMap[ts]
			if !ok {
				t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
			}

			if !present {
				break
			}
		}

		namesMap[ts] = true
	}

	for name, present := range namesMap {
		Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
	}
}
Esempio n. 6
0
func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
	sn.Lock()
	defer sn.Unlock()

	for id := range sn.repo.List(backend.Snapshot, ctx.Done()) {
		snapshot, err := restic.LoadSnapshot(sn.repo, id)
		if err != nil {
			return err
		}
		sn.knownSnapshots[snapshot.Time.Format(time.RFC3339)] = SnapshotWithId{snapshot, id}
	}
	return nil
}
Esempio n. 7
0
func loadTreeFromSnapshot(repo *repository.Repository, id backend.ID) (backend.ID, error) {
	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		debug.Log("Checker.loadTreeFromSnapshot", "error loading snapshot %v: %v", id.Str(), err)
		return backend.ID{}, err
	}

	if sn.Tree == nil {
		debug.Log("Checker.loadTreeFromSnapshot", "snapshot %v has no tree", id.Str())
		return backend.ID{}, fmt.Errorf("snapshot %v has no tree", id)
	}

	return *sn.Tree, nil
}
Esempio n. 8
0
func loadTreeFromSnapshot(repo restic.Repository, id restic.ID) (restic.ID, error) {
	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		debug.Log("error loading snapshot %v: %v", id.Str(), err)
		return restic.ID{}, err
	}

	if sn.Tree == nil {
		debug.Log("snapshot %v has no tree", id.Str())
		return restic.ID{}, errors.Errorf("snapshot %v has no tree", id)
	}

	return *sn.Tree, nil
}
Esempio n. 9
0
func printSnapshots(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	for id := range repo.List(backend.Snapshot, done) {
		snapshot, err := restic.LoadSnapshot(repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
			continue
		}

		fmt.Fprintf(wr, "snapshot_id: %v\n", id)

		err = prettyPrintJSON(wr, snapshot)
		if err != nil {
			return err
		}
	}

	return nil
}
Esempio n. 10
0
func runLs(gopts GlobalOptions, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errors.Fatalf("no snapshot ID given")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	snapshotIDString := args[0]
	var id restic.ID

	if snapshotIDString == "latest" {
		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
		if err != nil {
			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
		}
	} else {
		id, err = restic.FindSnapshot(repo, snapshotIDString)
		if err != nil {
			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
		}
	}

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)

	return printTree("", repo, *sn.Tree)
}
Esempio n. 11
0
func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error {
	debug.Log("restic.find", "searching in snapshot %s\n  for entries within [%s %s]", id.Str(), c.oldest, c.newest)

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	results, err := c.findInTree(repo, *sn.Tree, "")
	if err != nil {
		return err
	}

	if len(results) == 0 {
		return nil
	}
	c.global.Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
	for _, res := range results {
		res.node.Name = filepath.Join(res.path, res.node.Name)
		c.global.Printf("  %s\n", res.node)
	}

	return nil
}
Esempio n. 12
0
func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	results, err := findInTree(repo, pat, *sn.Tree, "")
	if err != nil {
		return err
	}

	if len(results) == 0 {
		return nil
	}
	Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
	for _, res := range results {
		res.node.Name = filepath.Join(res.path, res.node.Name)
		Printf("  %s\n", res.node)
	}

	return nil
}
Esempio n. 13
0
func (cmd CmdSnapshots) Execute(args []string) error {
	if len(args) != 0 {
		return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage())
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	tab := NewTable()
	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %s", "ID", "Date", "Source", "Directory")
	tab.RowFormat = "%-8s  %-19s  %-10s  %s"

	done := make(chan struct{})
	defer close(done)

	list := []*restic.Snapshot{}
	for id := range repo.List(backend.Snapshot, done) {
		sn, err := restic.LoadSnapshot(repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
			continue
		}

		pos := sort.Search(len(list), func(i int) bool {
			return list[i].Time.After(sn.Time)
		})

		if pos < len(list) {
			list = append(list, nil)
			copy(list[pos+1:], list[pos:])
			list[pos] = sn
		} else {
			list = append(list, sn)
		}
	}

	plen, err := repo.PrefixLength(backend.Snapshot)
	if err != nil {
		return err
	}

	for _, sn := range list {
		if len(sn.Paths) == 0 {
			continue
		}
		id := sn.ID()
		tab.Rows = append(tab.Rows, []interface{}{hex.EncodeToString(id[:plen/2]), sn.Time.Format(TimeFormat), sn.Hostname, sn.Paths[0]})

		if len(sn.Paths) > 1 {
			for _, path := range sn.Paths[1:] {
				tab.Rows = append(tab.Rows, []interface{}{"", "", "", path})
			}
		}
	}

	tab.Write(os.Stdout)

	return nil
}
Esempio n. 14
0
func TestMount(t *testing.T) {
	if !RunFuseTest {
		t.Skip("Skipping fuse tests")
	}

	checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) {
		snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots"))
		OK(t, err)
		namesInSnapshots, err := snapshotsDir.Readdirnames(-1)
		OK(t, err)
		Assert(t,
			len(namesInSnapshots) == len(snapshotIDs),
			"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))

		namesMap := make(map[string]bool)
		for _, name := range namesInSnapshots {
			namesMap[name] = false
		}

		for _, id := range snapshotIDs {
			snapshot, err := restic.LoadSnapshot(repo, id)
			OK(t, err)
			_, ok := namesMap[snapshot.Time.Format(time.RFC3339)]
			Assert(t, ok, "Snapshot %s isn't present in fuse dir", snapshot.Time.Format(time.RFC3339))
			namesMap[snapshot.Time.Format(time.RFC3339)] = true
		}
		for name, present := range namesMap {
			Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
		}
		OK(t, snapshotsDir.Close())
	}

	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
		cmdInit(t, global)
		repo, err := global.OpenRepository()
		OK(t, err)

		mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
		OK(t, err)

		// We remove the mountpoint now to check that cmdMount creates it
		RemoveAll(t, mountpoint)

		ready := make(chan struct{}, 2)
		done := make(chan struct{})
		go cmdMount(t, global, mountpoint, ready, done)
		<-ready
		defer close(done)
		OK(t, waitForMount(mountpoint))

		mountpointDir, err := os.Open(mountpoint)
		OK(t, err)
		names, err := mountpointDir.Readdirnames(-1)
		OK(t, err)
		Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`)
		OK(t, mountpointDir.Close())

		checkSnapshots(repo, mountpoint, []backend.ID{})

		datafile := filepath.Join("testdata", "backup-data.tar.gz")
		fd, err := os.Open(datafile)
		if os.IsNotExist(err) {
			t.Skipf("unable to find data file %q, skipping", datafile)
			return
		}
		OK(t, err)
		OK(t, fd.Close())

		SetupTarTestFixture(t, env.testdata, datafile)

		// first backup
		cmdBackup(t, global, []string{env.testdata}, nil)
		snapshotIDs := cmdList(t, global, "snapshots")
		Assert(t, len(snapshotIDs) == 1,
			"expected one snapshot, got %v", snapshotIDs)

		checkSnapshots(repo, mountpoint, snapshotIDs)

		// second backup, implicit incremental
		cmdBackup(t, global, []string{env.testdata}, nil)
		snapshotIDs = cmdList(t, global, "snapshots")
		Assert(t, len(snapshotIDs) == 2,
			"expected two snapshots, got %v", snapshotIDs)

		checkSnapshots(repo, mountpoint, snapshotIDs)

		// third backup, explicit incremental
		cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
		snapshotIDs = cmdList(t, global, "snapshots")
		Assert(t, len(snapshotIDs) == 3,
			"expected three snapshots, got %v", snapshotIDs)

		checkSnapshots(repo, mountpoint, snapshotIDs)
	})
}
Esempio n. 15
0
// Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
// used to compare the files to the ones archived at the time this snapshot was
// taken.
func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
	paths = unique(paths)
	sort.Sort(baseNameSlice(paths))

	debug.Log("start for %v", paths)

	debug.RunHook("Archiver.Snapshot", nil)

	// signal the whole pipeline to stop
	done := make(chan struct{})
	var err error

	p.Start()
	defer p.Done()

	// create new snapshot
	sn, err := restic.NewSnapshot(paths, tags)
	if err != nil {
		return nil, restic.ID{}, err
	}
	sn.Excludes = arch.Excludes

	jobs := archivePipe{}

	// use parent snapshot (if some was given)
	if parentID != nil {
		sn.Parent = parentID

		// load parent snapshot
		parent, err := restic.LoadSnapshot(arch.repo, *parentID)
		if err != nil {
			return nil, restic.ID{}, err
		}

		// start walker on old tree
		ch := make(chan walk.TreeJob)
		go walk.Tree(arch.repo, *parent.Tree, done, ch)
		jobs.Old = ch
	} else {
		// use closed channel
		ch := make(chan walk.TreeJob)
		close(ch)
		jobs.Old = ch
	}

	// start walker
	pipeCh := make(chan pipe.Job)
	resCh := make(chan pipe.Result, 1)
	go func() {
		pipe.Walk(paths, arch.SelectFilter, done, pipeCh, resCh)
		debug.Log("pipe.Walk done")
	}()
	jobs.New = pipeCh

	ch := make(chan pipe.Job)
	go jobs.compare(done, ch)

	var wg sync.WaitGroup
	entCh := make(chan pipe.Entry)
	dirCh := make(chan pipe.Dir)

	// split
	wg.Add(1)
	go func() {
		pipe.Split(ch, dirCh, entCh)
		debug.Log("split done")
		close(dirCh)
		close(entCh)
		wg.Done()
	}()

	// run workers
	for i := 0; i < maxConcurrency; i++ {
		wg.Add(2)
		go arch.fileWorker(&wg, p, done, entCh)
		go arch.dirWorker(&wg, p, done, dirCh)
	}

	// run index saver
	var wgIndexSaver sync.WaitGroup
	stopIndexSaver := make(chan struct{})
	wgIndexSaver.Add(1)
	go arch.saveIndexes(&wgIndexSaver, stopIndexSaver)

	// wait for all workers to terminate
	debug.Log("wait for workers")
	wg.Wait()

	// stop index saver
	close(stopIndexSaver)
	wgIndexSaver.Wait()

	debug.Log("workers terminated")

	// receive the top-level tree
	root := (<-resCh).(*restic.Node)
	debug.Log("root node received: %v", root.Subtree.Str())
	sn.Tree = root.Subtree

	// save snapshot
	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
	if err != nil {
		return nil, restic.ID{}, err
	}

	debug.Log("saved snapshot %v", id.Str())

	// flush repository
	err = arch.repo.Flush()
	if err != nil {
		return nil, restic.ID{}, err
	}

	// save index
	err = arch.repo.SaveIndex()
	if err != nil {
		debug.Log("error saving index: %v", err)
		return nil, restic.ID{}, err
	}

	debug.Log("saved indexes")

	return sn, id, nil
}
Esempio n. 16
0
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatalf("wrong number of arguments")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	tab := NewTable()
	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"

	done := make(chan struct{})
	defer close(done)

	list := []*restic.Snapshot{}
	for id := range repo.List(restic.SnapshotFile, done) {
		sn, err := restic.LoadSnapshot(repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
			continue
		}

		if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
			pos := sort.Search(len(list), func(i int) bool {
				return list[i].Time.After(sn.Time)
			})

			if pos < len(list) {
				list = append(list, nil)
				copy(list[pos+1:], list[pos:])
				list[pos] = sn
			} else {
				list = append(list, sn)
			}
		}

	}

	for _, sn := range list {
		if len(sn.Paths) == 0 {
			continue
		}

		firstTag := ""
		if len(sn.Tags) > 0 {
			firstTag = sn.Tags[0]
		}

		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})

		rows := len(sn.Paths)
		if len(sn.Tags) > rows {
			rows = len(sn.Tags)
		}

		for i := 1; i < rows; i++ {
			path := ""
			if len(sn.Paths) > i {
				path = sn.Paths[i]
			}

			tag := ""
			if len(sn.Tags) > i {
				tag = sn.Tags[i]
			}

			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
		}
	}

	tab.Write(os.Stdout)

	return nil
}