func TestCreateSnapshot(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
for i := 0; i < testCreateSnapshots; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
}
snapshots, err := restic.LoadAllSnapshots(repo)
if err != nil {
t.Fatal(err)
}
if len(snapshots) != testCreateSnapshots {
t.Fatalf("got %d snapshots, expected %d", len(snapshots), 1)
}
sn := snapshots[0]
if sn.Time.Before(testSnapshotTime) || sn.Time.After(testSnapshotTime.Add(testCreateSnapshots*time.Second)) {
t.Fatalf("timestamp %v is outside of the allowed time range", sn.Time)
}
if sn.Tree == nil {
t.Fatalf("tree id is nil")
}
if sn.Tree.IsNull() {
t.Fatalf("snapshot has zero tree ID")
}
checker.TestCheckRepo(t, repo)
}
func TestCreateSnapshot(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
for i := 0; i < testCreateSnapshots; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second))
}
snapshots, err := restic.LoadAllSnapshots(repo)
if err != nil {
t.Fatal(err)
}
if len(snapshots) != testCreateSnapshots {
t.Fatalf("got %d snapshots, expected %d", len(snapshots), 1)
}
sn := snapshots[0]
if sn.Time.Before(testSnapshotTime) || sn.Time.After(testSnapshotTime.Add(testCreateSnapshots*time.Second)) {
t.Fatalf("timestamp %v is outside of the allowed time range", sn.Time)
}
if sn.Tree == nil {
t.Fatalf("tree id is nil")
}
if sn.Tree.IsNull() {
t.Fatalf("snapshot has zero tree ID")
}
chkr := checker.New(repo)
hints, errs := chkr.LoadIndex()
if len(errs) != 0 {
t.Fatalf("errors loading index: %v", errs)
}
if len(hints) != 0 {
t.Fatalf("errors loading index: %v", hints)
}
done := make(chan struct{})
defer close(done)
errChan := make(chan error)
go chkr.Structure(errChan, done)
for err := range errChan {
t.Error(err)
}
errChan = make(chan error)
go chkr.ReadData(nil, errChan, done)
for err := range errChan {
t.Error(err)
}
}
func runPrune(gopts GlobalOptions) error {
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
done := make(chan struct{})
defer close(done)
var stats struct {
blobs int
packs int
snapshots int
bytes int64
}
Verbosef("counting files in repo\n")
for _ = range repo.List(restic.DataFile, done) {
stats.packs++
}
Verbosef("building new index for repo\n")
bar := newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
idx, err := index.New(repo, bar)
if err != nil {
return err
}
for _, pack := range idx.Packs {
stats.bytes += pack.Size
}
Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
blobCount := make(map[restic.BlobHandle]int)
duplicateBlobs := 0
duplicateBytes := 0
// find duplicate blobs
for _, p := range idx.Packs {
for _, entry := range p.Entries {
stats.blobs++
h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
blobCount[h]++
if blobCount[h] > 1 {
duplicateBlobs++
duplicateBytes += int(entry.Length)
}
}
}
Verbosef("processed %d blobs: %d duplicate blobs, %v duplicate\n",
stats.blobs, duplicateBlobs, formatBytes(uint64(duplicateBytes)))
Verbosef("load all snapshots\n")
// find referenced blobs
snapshots, err := restic.LoadAllSnapshots(repo)
if err != nil {
return err
}
stats.snapshots = len(snapshots)
Verbosef("find data that is still in use for %d snapshots\n", stats.snapshots)
usedBlobs := restic.NewBlobSet()
seenBlobs := restic.NewBlobSet()
bar = newProgressMax(!gopts.Quiet, uint64(len(snapshots)), "snapshots")
bar.Start()
for _, sn := range snapshots {
debug.Log("process snapshot %v", sn.ID().Str())
err = restic.FindUsedBlobs(repo, *sn.Tree, usedBlobs, seenBlobs)
if err != nil {
return err
}
debug.Log("found %v blobs for snapshot %v", sn.ID().Str())
bar.Report(restic.Stat{Blobs: 1})
}
bar.Done()
Verbosef("found %d of %d data blobs still in use, removing %d blobs\n",
len(usedBlobs), stats.blobs, stats.blobs-len(usedBlobs))
// find packs that need a rewrite
rewritePacks := restic.NewIDSet()
for h, blob := range idx.Blobs {
if !usedBlobs.Has(h) {
rewritePacks.Merge(blob.Packs)
continue
}
if blobCount[h] > 1 {
rewritePacks.Merge(blob.Packs)
}
}
removeBytes := 0
// find packs that are unneeded
removePacks := restic.NewIDSet()
for packID, p := range idx.Packs {
hasActiveBlob := false
for _, blob := range p.Entries {
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
if usedBlobs.Has(h) {
hasActiveBlob = true
continue
}
removeBytes += int(blob.Length)
}
if hasActiveBlob {
continue
}
removePacks.Insert(packID)
if !rewritePacks.Has(packID) {
return errors.Fatalf("pack %v is unneeded, but not contained in rewritePacks", packID.Str())
}
rewritePacks.Delete(packID)
}
Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
err = repository.Repack(repo, rewritePacks, usedBlobs)
if err != nil {
return err
}
for packID := range removePacks {
err = repo.Backend().Remove(restic.DataFile, packID.String())
if err != nil {
Warnf("unable to remove file %v from the repository\n", packID.Str())
}
}
Verbosef("creating new index\n")
stats.packs = 0
for _ = range repo.List(restic.DataFile, done) {
stats.packs++
}
bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
idx, err = index.New(repo, bar)
if err != nil {
return err
}
var supersedes restic.IDs
for idxID := range repo.List(restic.IndexFile, done) {
err := repo.Backend().Remove(restic.IndexFile, idxID.String())
if err != nil {
fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
}
supersedes = append(supersedes, idxID)
}
id, err := idx.Save(repo, supersedes)
if err != nil {
return err
}
Verbosef("saved new index as %v\n", id.Str())
Verbosef("done\n")
return nil
}
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
// first, process all snapshot IDs given as arguments
for _, s := range args {
id, err := restic.FindSnapshot(repo, s)
if err != nil {
return err
}
if !opts.DryRun {
err = repo.Backend().Remove(restic.SnapshotFile, id.String())
if err != nil {
return err
}
Verbosef("removed snapshot %v\n", id.Str())
} else {
Verbosef("would removed snapshot %v\n", id.Str())
}
}
policy := restic.ExpirePolicy{
Last: opts.Last,
Hourly: opts.Hourly,
Daily: opts.Daily,
Weekly: opts.Weekly,
Monthly: opts.Monthly,
Yearly: opts.Yearly,
Tags: opts.KeepTags,
}
if policy.Empty() {
return nil
}
// then, load all remaining snapshots
snapshots, err := restic.LoadAllSnapshots(repo)
if err != nil {
return err
}
// group by hostname and dirs
type key struct {
Hostname string
Dirs string
}
snapshotGroups := make(map[key]restic.Snapshots)
for _, sn := range snapshots {
if opts.Hostname != "" && sn.Hostname != opts.Hostname {
continue
}
if !sn.HasTags(opts.Tags) {
continue
}
k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")}
list := snapshotGroups[k]
list = append(list, sn)
snapshotGroups[k] = list
}
for key, snapshotGroup := range snapshotGroups {
Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs)
keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
Printf("keep %d snapshots:\n", len(keep))
printSnapshots(globalOptions.stdout, keep)
Printf("\n")
Printf("remove %d snapshots:\n", len(remove))
printSnapshots(globalOptions.stdout, remove)
Printf("\n")
if !opts.DryRun {
for _, sn := range remove {
err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
if err != nil {
return err
}
}
}
}
return nil
}