Beispiel #1
0
func TestUnreferencedBlobs(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
		OK(t, repo.Backend().Remove(backend.Snapshot, snID))

		unusedBlobsBySnapshot := backend.IDs{
			ParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
			ParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"),
			ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"),
			ParseID("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7"),
			ParseID("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d"),
			ParseID("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10"),
		}

		sort.Sort(unusedBlobsBySnapshot)

		chkr := checker.New(repo)
		OK(t, chkr.LoadIndex())
		OKs(t, checkPacks(chkr))
		OKs(t, checkStruct(chkr))

		blobs := chkr.UnusedBlobs()
		sort.Sort(blobs)

		Equals(t, unusedBlobsBySnapshot, blobs)
	})
}
Beispiel #2
0
func TestMissingPack(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
		OK(t, repo.Backend().Remove(backend.Data, packID))

		chkr := checker.New(repo)
		hints, errs := chkr.LoadIndex()
		if len(errs) > 0 {
			t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
		}

		if len(hints) > 0 {
			t.Errorf("expected no hints, got %v: %v", len(hints), hints)
		}

		errs = checkPacks(chkr)

		Assert(t, len(errs) == 1,
			"expected exactly one error, got %v", len(errs))

		if err, ok := errs[0].(checker.PackError); ok {
			Equals(t, packID, err.ID.String())
		} else {
			t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
		}
	})
}
Beispiel #3
0
func TestDuplicatePacksInIndex(t *testing.T) {
	WithTestEnvironment(t, checkerDuplicateIndexTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		chkr := checker.New(repo)
		hints, errs := chkr.LoadIndex()
		if len(hints) == 0 {
			t.Fatalf("did not get expected checker hints for duplicate packs in indexes")
		}

		found := false
		for _, hint := range hints {
			if _, ok := hint.(checker.ErrDuplicatePacks); ok {
				found = true
			} else {
				t.Errorf("got unexpected hint: %v", hint)
			}
		}

		if !found {
			t.Fatalf("did not find hint ErrDuplicatePacks")
		}

		if len(errs) > 0 {
			t.Errorf("expected no errors, got %v: %v", len(errs), errs)
		}

	})
}
Beispiel #4
0
func TestUnreferencedPack(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		// index 3f1a only references pack 60e0
		indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
		packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
		OK(t, repo.Backend().Remove(backend.Index, indexID))

		chkr := checker.New(repo)
		hints, errs := chkr.LoadIndex()
		if len(errs) > 0 {
			t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
		}

		if len(hints) > 0 {
			t.Errorf("expected no hints, got %v: %v", len(hints), hints)
		}

		errs = checkPacks(chkr)

		Assert(t, len(errs) == 1,
			"expected exactly one error, got %v", len(errs))

		if err, ok := errs[0].(checker.PackError); ok {
			Equals(t, packID, err.ID.String())
		} else {
			t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
		}
	})
}
Beispiel #5
0
func (cmd CmdOptimize) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("optimize has no arguments")
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("Create exclusive lock for repository\n")
	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	chkr := checker.New(repo)

	cmd.global.Verbosef("Load indexes\n")
	_, errs := chkr.LoadIndex()

	if len(errs) > 0 {
		for _, err := range errs {
			cmd.global.Warnf("error: %v\n", err)
		}
		return fmt.Errorf("LoadIndex returned errors")
	}

	done := make(chan struct{})
	errChan := make(chan error)
	go chkr.Structure(errChan, done)

	for err := range errChan {
		if e, ok := err.(checker.TreeError); ok {
			cmd.global.Warnf("error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				cmd.global.Warnf("  %v\n", treeErr)
			}
		} else {
			cmd.global.Warnf("error: %v\n", err)
		}
	}

	unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...)
	cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs))

	repacker := checker.NewRepacker(repo, unusedBlobs)
	err = repacker.Repack()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("repacking done\n")
	return nil
}
Beispiel #6
0
func TestCheckRepo(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		chkr := checker.New(repo)
		OK(t, chkr.LoadIndex())
		OKs(t, checkPacks(chkr))
		OKs(t, checkStruct(chkr))
	})
}
Beispiel #7
0
func TestRepacker(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)
		OK(t, repo.LoadIndex())

		repo.Backend().Remove(backend.Snapshot, "c2b53c5e6a16db92fbb9aa08bd2794c58b379d8724d661ee30d20898bdfdff22")

		unusedBlobs := backend.IDSet{
			ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{},
			ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{},
			ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{},
			ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{},
		}

		chkr := checker.New(repo)
		_, errs := chkr.LoadIndex()
		OKs(t, errs)

		errs = checkStruct(chkr)
		OKs(t, errs)

		list := backend.NewIDSet(chkr.UnusedBlobs()...)
		if !unusedBlobs.Equals(list) {
			t.Fatalf("expected unused blobs:\n  %v\ngot:\n  %v", unusedBlobs, list)
		}

		repacker := checker.NewRepacker(repo, unusedBlobs)
		OK(t, repacker.Repack())

		chkr = checker.New(repo)
		_, errs = chkr.LoadIndex()
		OKs(t, errs)
		OKs(t, checkPacks(chkr))
		OKs(t, checkStruct(chkr))

		blobs := chkr.UnusedBlobs()
		Assert(t, len(blobs) == 0,
			"expected zero unused blobs, got %v", blobs)
	})
}
Beispiel #8
0
func createAndInitChecker(t *testing.T, repo *repository.Repository) *checker.Checker {
	chkr := checker.New(repo)

	hints, errs := chkr.LoadIndex()
	if len(errs) > 0 {
		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
	}

	if len(hints) > 0 {
		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
	}

	return chkr
}
Beispiel #9
0
func TestCheckerModifiedData(t *testing.T) {
	be := mem.New()

	repo := repository.New(be)
	OK(t, repo.Init(TestPassword))

	arch := restic.NewArchiver(repo)
	_, id, err := arch.Snapshot(nil, []string{"."}, nil)
	OK(t, err)
	t.Logf("archived as %v", id.Str())

	beError := &errorBackend{Backend: be}
	checkRepo := repository.New(beError)
	OK(t, checkRepo.SearchKey(TestPassword))

	chkr := checker.New(checkRepo)

	hints, errs := chkr.LoadIndex()
	if len(errs) > 0 {
		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
	}

	if len(hints) > 0 {
		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
	}

	beError.ProduceErrors = true
	errFound := false
	for _, err := range checkPacks(chkr) {
		t.Logf("pack error: %v", err)
	}

	for _, err := range checkStruct(chkr) {
		t.Logf("struct error: %v", err)
	}

	for _, err := range checkData(chkr) {
		t.Logf("struct error: %v", err)
		errFound = true
	}

	if !errFound {
		t.Fatal("no error found, checker is broken")
	}
}
Beispiel #10
0
func TestCheckRepo(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		chkr := checker.New(repo)
		hints, errs := chkr.LoadIndex()
		if len(errs) > 0 {
			t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
		}

		if len(hints) > 0 {
			t.Errorf("expected no hints, got %v: %v", len(hints), hints)
		}

		OKs(t, checkPacks(chkr))
		OKs(t, checkStruct(chkr))
	})
}
Beispiel #11
0
func TestUnreferencedPack(t *testing.T) {
	WithTestEnvironment(t, checkerTestData, func(repodir string) {
		repo := OpenLocalRepo(t, repodir)

		// index 8eb5 only references pack 60e0
		indexID := "8eb5b61062bf8e959f244fba0c971108bc8d4d2a4b236f71a704998e28cc5cf6"
		packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
		OK(t, repo.Backend().Remove(backend.Index, indexID))

		chkr := checker.New(repo)
		OK(t, chkr.LoadIndex())
		errs := checkPacks(chkr)

		Assert(t, len(errs) == 1,
			"expected exactly one error, got %v", len(errs))

		if err, ok := errs[0].(checker.PackError); ok {
			Equals(t, packID, err.ID.String())
		} else {
			t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
		}
	})
}
Beispiel #12
0
func (cmd CmdCheck) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("check has no arguments")
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("Create exclusive lock for repository\n")
	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	chkr := checker.New(repo)

	cmd.global.Verbosef("Load indexes\n")
	hints, errs := chkr.LoadIndex()

	dupFound := false
	for _, hint := range hints {
		cmd.global.Printf("%v\n", hint)
		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
			dupFound = true
		}
	}

	if dupFound {
		cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n")
	}

	if len(errs) > 0 {
		for _, err := range errs {
			cmd.global.Warnf("error: %v\n", err)
		}
		return fmt.Errorf("LoadIndex returned errors")
	}

	done := make(chan struct{})
	defer close(done)

	errorsFound := false
	errChan := make(chan error)

	cmd.global.Verbosef("Check all packs\n")
	go chkr.Packs(errChan, done)

	foundOrphanedPacks := false
	for err := range errChan {
		errorsFound = true
		fmt.Fprintf(os.Stderr, "%v\n", err)

		if e, ok := err.(checker.PackError); ok && e.Orphaned {
			foundOrphanedPacks = true
		}
	}

	cmd.global.Verbosef("Check snapshots, trees and blobs\n")
	errChan = make(chan error)
	go chkr.Structure(errChan, done)

	for err := range errChan {
		errorsFound = true
		if e, ok := err.(checker.TreeError); ok {
			fmt.Fprintf(os.Stderr, "error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				fmt.Fprintf(os.Stderr, "  %v\n", treeErr)
			}
		} else {
			fmt.Fprintf(os.Stderr, "error: %v\n", err)
		}
	}

	for _, id := range chkr.UnusedBlobs() {
		cmd.global.Verbosef("unused blob %v\n", id.Str())
	}

	if foundOrphanedPacks && cmd.RemoveOrphaned {
		IDs := chkr.OrphanedPacks()
		cmd.global.Verbosef("Remove %d orphaned packs... ", len(IDs))

		for _, id := range IDs {
			if err := repo.Backend().Remove(backend.Data, id.String()); err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
			}
		}

		cmd.global.Verbosef("done\n")
	}

	if errorsFound {
		return errors.New("repository contains errors")
	}
	return nil
}
Beispiel #13
0
func (cmd CmdCheck) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("check has no arguments")
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	if !cmd.global.NoLock {
		cmd.global.Verbosef("Create exclusive lock for repository\n")
		lock, err := lockRepoExclusive(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	chkr := checker.New(repo)

	cmd.global.Verbosef("Load indexes\n")
	hints, errs := chkr.LoadIndex()

	dupFound := false
	for _, hint := range hints {
		cmd.global.Printf("%v\n", hint)
		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
			dupFound = true
		}
	}

	if dupFound {
		cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n")
	}

	if len(errs) > 0 {
		for _, err := range errs {
			cmd.global.Warnf("error: %v\n", err)
		}
		return fmt.Errorf("LoadIndex returned errors")
	}

	done := make(chan struct{})
	defer close(done)

	errorsFound := false
	errChan := make(chan error)

	cmd.global.Verbosef("Check all packs\n")
	go chkr.Packs(errChan, done)

	for err := range errChan {
		errorsFound = true
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}

	cmd.global.Verbosef("Check snapshots, trees and blobs\n")
	errChan = make(chan error)
	go chkr.Structure(errChan, done)

	for err := range errChan {
		errorsFound = true
		if e, ok := err.(checker.TreeError); ok {
			fmt.Fprintf(os.Stderr, "error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				fmt.Fprintf(os.Stderr, "  %v\n", treeErr)
			}
		} else {
			fmt.Fprintf(os.Stderr, "error: %v\n", err)
		}
	}

	if cmd.CheckUnused {
		for _, id := range chkr.UnusedBlobs() {
			cmd.global.Verbosef("unused blob %v\n", id.Str())
			errorsFound = true
		}
	}

	if cmd.ReadData {
		cmd.global.Verbosef("Read all data\n")

		p := cmd.newReadProgress(restic.Stat{Blobs: chkr.CountPacks()})
		errChan := make(chan error)

		go chkr.ReadData(p, errChan, done)

		for err := range errChan {
			errorsFound = true
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}
	}

	if errorsFound {
		return errors.New("repository contains errors")
	}
	return nil
}
Beispiel #14
0
func (cmd CmdCheck) Execute(args []string) error {
	if len(args) != 0 {
		return errors.New("check has no arguments")
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}

	cmd.global.Verbosef("Create exclusive lock for repository\n")
	lock, err := lockRepoExclusive(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	chkr := checker.New(repo)

	cmd.global.Verbosef("Load indexes\n")
	if err = chkr.LoadIndex(); err != nil {
		return err
	}

	done := make(chan struct{})
	defer close(done)

	errorsFound := false
	errChan := make(chan error)

	cmd.global.Verbosef("Check all packs\n")
	go chkr.Packs(errChan, done)

	foundOrphanedPacks := false
	for err := range errChan {
		errorsFound = true
		fmt.Fprintf(os.Stderr, "%v\n", err)

		if e, ok := err.(checker.PackError); ok && e.Orphaned {
			foundOrphanedPacks = true
		}
	}

	cmd.global.Verbosef("Check snapshots, trees and blobs\n")
	errChan = make(chan error)
	go chkr.Structure(errChan, done)

	for err := range errChan {
		errorsFound = true
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
	}

	for _, id := range chkr.UnusedBlobs() {
		cmd.global.Verbosef("unused blob %v\n", id.Str())
	}

	if foundOrphanedPacks && cmd.RemoveOrphaned {
		IDs := chkr.OrphanedPacks()
		cmd.global.Verbosef("Remove %d orphaned packs... ", len(IDs))

		for _, id := range IDs {
			if err := repo.Backend().Remove(backend.Data, id.String()); err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
			}
		}

		cmd.global.Verbosef("done\n")
	}

	if errorsFound {
		return errors.New("repository contains errors")
	}
	return nil
}