func packIDTester(repo *repository.Repository, inChan <-chan backend.ID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) {
debug.Log("Checker.testPackID", "worker start")
defer debug.Log("Checker.testPackID", "worker done")
defer wg.Done()
for id := range inChan {
ok, err := repo.Backend().Test(backend.Data, id.String())
if err != nil {
err = PackError{ID: id, Err: err}
} else {
if !ok {
err = PackError{ID: id, Err: errors.New("does not exist")}
}
}
if err != nil {
debug.Log("Checker.testPackID", "error checking for pack %s: %v", id.Str(), err)
select {
case <-done:
return
case errChan <- err:
}
continue
}
debug.Log("Checker.testPackID", "pack %s exists", id.Str())
}
}
func (node Node) createFileAt(path string, repo *repository.Repository) error {
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()
if err != nil {
return errors.Annotate(err, "OpenFile")
}
var buf []byte
for _, id := range node.Content {
_, _, _, length, err := repo.Index().Lookup(id)
if err != nil {
return err
}
buf = buf[:cap(buf)]
if uint(len(buf)) < length {
buf = make([]byte, length)
}
buf, err := repo.LoadBlob(pack.Data, id, buf)
if err != nil {
return errors.Annotate(err, "Load")
}
_, err = f.Write(buf)
if err != nil {
return errors.Annotate(err, "Write")
}
}
return nil
}
func printTrees(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
trees := []backend.ID{}
for _, idx := range repo.Index().All() {
for blob := range idx.Each(nil) {
if blob.Type != pack.Tree {
continue
}
trees = append(trees, blob.ID)
}
}
for _, id := range trees {
tree, err := restic.LoadTree(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err)
continue
}
fmt.Fprintf(wr, "tree_id: %v\n", id)
prettyPrintJSON(wr, tree)
}
return nil
}
func findLatestSnapshot(repo *repository.Repository, targets []string) (backend.ID, error) {
var (
latest time.Time
latestID backend.ID
found bool
)
for snapshotID := range repo.List(backend.Snapshot, make(chan struct{})) {
snapshot, err := restic.LoadSnapshot(repo, snapshotID)
if err != nil {
return backend.ID{}, fmt.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && samePaths(snapshot.Paths, targets) {
latest = snapshot.Time
latestID = snapshotID
found = true
}
}
if !found {
return backend.ID{}, errNoSnapshotFound
}
return latestID, nil
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func FindSnapshot(repo *repository.Repository, s string) (backend.ID, error) {
// find snapshot id with prefix
name, err := backend.Find(repo.Backend(), backend.Snapshot, s)
if err != nil {
return nil, err
}
return backend.ParseID(name)
}
func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error) {
sn := &Snapshot{id: id}
err := repo.LoadJSONUnpacked(backend.Snapshot, id, sn)
if err != nil {
return nil, err
}
return sn, nil
}
// LoadLock loads and unserializes a lock from a repository.
func LoadLock(repo *repository.Repository, id backend.ID) (*Lock, error) {
lock := &Lock{}
if err := repo.LoadJSONUnpacked(backend.Lock, id, lock); err != nil {
return nil, err
}
lock.lockID = &id
return lock, nil
}
func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) {
hostname, err := os.Hostname()
if err != nil {
return backend.ID{}, err
}
newLock := &restic.Lock{Time: t, PID: pid, Hostname: hostname}
return repo.SaveJSONUnpacked(backend.Lock, &newLock)
}
func TeardownRepo(t testing.TB, repo *repository.Repository) {
if !TestCleanup {
l := repo.Backend().(*local.Local)
t.Logf("leaving local backend at %s\n", l.Location())
return
}
OK(t, repo.Delete())
}
func LoadTree(repo *repository.Repository, id backend.ID) (*Tree, error) {
tree := &Tree{}
err := repo.LoadJSONPack(pack.Tree, id, tree)
if err != nil {
return nil, err
}
return tree, nil
}
func (cmd CmdKey) addKey(repo *repository.Repository) error {
id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
if err != nil {
return fmt.Errorf("creating new key failed: %v\n", err)
}
cmd.global.Verbosef("saved new key as %s\n", id)
return nil
}
func list(repo *repository.Repository, t backend.Type) (IDs []string) {
done := make(chan struct{})
defer close(done)
for id := range repo.List(t, done) {
IDs = append(IDs, id.String())
}
return IDs
}
// FindBlobsForPacks returns the set of blobs contained in a pack of packs.
func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) {
blobs := backend.NewIDSet()
for packID := range packs {
for _, packedBlob := range repo.Index().ListPack(packID) {
blobs.Insert(packedBlob.ID)
}
}
return blobs, nil
}
func TeardownRepo(repo *repository.Repository) {
if !TestCleanup {
l := repo.Backend().(*local.Local)
fmt.Printf("leaving local backend at %s\n", l.Location())
return
}
err := repo.Delete()
if err != nil {
panic(err)
}
}
// saveRandomDataBlobs generates random data blobs and saves them to the repository.
func saveRandomDataBlobs(t testing.TB, repo *repository.Repository, num int, sizeMax int) {
for i := 0; i < num; i++ {
size := mrand.Int() % sizeMax
buf := make([]byte, size)
_, err := io.ReadFull(rand.Reader, buf)
OK(t, err)
_, err = repo.SaveAndEncrypt(pack.Data, buf, nil)
OK(t, err)
}
}
func (cmd CmdKey) deleteKey(repo *repository.Repository, name string) error {
if name == repo.KeyName() {
return errors.New("refusing to remove key currently used to access repository")
}
err := repo.Backend().Remove(backend.Key, name)
if err != nil {
return err
}
cmd.global.Verbosef("removed key %v\n", name)
return nil
}
// RemoveStaleLocks deletes all locks detected as stale from the repository.
func RemoveStaleLocks(repo *repository.Repository) error {
return eachLock(repo, func(id backend.ID, lock *Lock, err error) error {
// ignore locks that cannot be loaded
if err != nil {
return nil
}
if lock.Stale() {
return repo.Backend().Remove(backend.Lock, id.String())
}
return nil
})
}
func eachLock(repo *repository.Repository, f func(backend.ID, *Lock, error) error) error {
done := make(chan struct{})
defer close(done)
for id := range repo.List(backend.Lock, done) {
lock, err := LoadLock(repo, id)
err = f(id, lock, err)
if err != nil {
return err
}
}
return nil
}
// FindPacksForBlobs returns the set of packs that contain the blobs.
func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) {
packs := backend.NewIDSet()
idx := repo.Index()
for id := range blobs {
blob, err := idx.Lookup(id)
if err != nil {
return nil, err
}
packs.Insert(blob.PackID)
}
return packs, nil
}
// RepackBlobs reads all blobs in blobIDs from src and saves them into new pack
// files in dst. Source and destination repo may be the same.
func RepackBlobs(src, dst *repository.Repository, blobIDs backend.IDSet) (err error) {
for id := range blobIDs {
err = repackBlob(src, dst, id)
if err != nil {
return err
}
}
err = dst.Flush()
if err != nil {
return err
}
return nil
}
func newFile(repo *repository.Repository, node *restic.Node) (*file, error) {
sizes := make([]uint32, len(node.Content))
for i, blobID := range node.Content {
length, err := repo.Index().LookupSize(blobID)
if err != nil {
return nil, err
}
sizes[i] = uint32(length)
}
return &file{
repo: repo,
node: node,
sizes: sizes,
blobs: make([][]byte, len(node.Content)),
}, nil
}
// loadSnapshotTreeIDs loads all snapshots from backend and returns the tree IDs.
func loadSnapshotTreeIDs(repo *repository.Repository) (backend.IDs, []error) {
var trees struct {
IDs backend.IDs
sync.Mutex
}
var errs struct {
errs []error
sync.Mutex
}
snapshotWorker := func(strID string, done <-chan struct{}) error {
id, err := backend.ParseID(strID)
if err != nil {
return err
}
debug.Log("Checker.Snaphots", "load snapshot %v", id.Str())
treeID, err := loadTreeFromSnapshot(repo, id)
if err != nil {
errs.Lock()
errs.errs = append(errs.errs, err)
errs.Unlock()
return nil
}
debug.Log("Checker.Snaphots", "snapshot %v has tree %v", id.Str(), treeID.Str())
trees.Lock()
trees.IDs = append(trees.IDs, treeID)
trees.Unlock()
return nil
}
err := repository.FilesInParallel(repo.Backend(), backend.Snapshot, defaultParallelism, snapshotWorker)
if err != nil {
errs.errs = append(errs.errs, err)
}
return trees.IDs, errs.errs
}
// Clear removes information from the cache that isn't present in the repository any more.
func (c *Cache) Clear(repo *repository.Repository) error {
list, err := c.list(backend.Snapshot)
if err != nil {
return err
}
for _, entry := range list {
debug.Log("Cache.Clear", "found entry %v", entry)
if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)
err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)
if err != nil {
return err
}
}
}
return nil
}
func printSnapshots(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
for id := range repo.List(backend.Snapshot, done) {
snapshot, err := restic.LoadSnapshot(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
continue
}
fmt.Fprintf(wr, "snapshot_id: %v\n", id)
err = prettyPrintJSON(wr, snapshot)
if err != nil {
return err
}
}
return nil
}
func (cmd CmdKey) listKeys(s *repository.Repository) error {
tab := NewTable()
tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created")
tab.RowFormat = "%s%-10s %-10s %-10s %s"
plen, err := s.PrefixLength(backend.Key)
if err != nil {
return err
}
done := make(chan struct{})
defer close(done)
for id := range s.List(backend.Key, done) {
k, err := repository.LoadKey(s, id.String())
if err != nil {
cmd.global.Warnf("LoadKey() failed: %v\n", err)
continue
}
var current string
if id.String() == s.KeyName() {
current = "*"
} else {
current = " "
}
tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen],
k.Username, k.Hostname, k.Created.Format(TimeFormat)})
}
return tab.Write(cmd.global.stdout)
}
// repackBlob loads a single blob from src and saves it in dst.
func repackBlob(src, dst *repository.Repository, id backend.ID) error {
blob, err := src.Index().Lookup(id)
if err != nil {
return err
}
debug.Log("RepackBlobs", "repacking blob %v, len %v", id.Str(), blob.PlaintextLength())
buf := make([]byte, 0, blob.PlaintextLength())
buf, err = src.LoadBlob(blob.Type, id, buf)
if err != nil {
return err
}
if uint(len(buf)) != blob.PlaintextLength() {
debug.Log("RepackBlobs", "repack blob %v: len(buf) isn't equal to length: %v = %v", id.Str(), len(buf), blob.PlaintextLength())
return errors.New("LoadBlob returned wrong data, len() doesn't match")
}
_, err = dst.SaveAndEncrypt(blob.Type, buf, &id)
if err != nil {
return err
}
return nil
}
func (node Node) createFileAt(path string, repo *repository.Repository) error {
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()
if err != nil {
return errors.Annotate(err, "OpenFile")
}
for _, id := range node.Content {
buf, err := repo.LoadBlob(pack.Data, id)
if err != nil {
return errors.Annotate(err, "Load")
}
_, err = f.Write(buf)
if err != nil {
return errors.Annotate(err, "Write")
}
}
return nil
}
func (cmd CmdKey) changePassword(repo *repository.Repository) error {
id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
if err != nil {
return fmt.Errorf("creating new key failed: %v\n", err)
}
err = repo.Backend().Remove(backend.Key, repo.KeyName())
if err != nil {
return err
}
cmd.global.Verbosef("saved new key as %s\n", id)
return nil
}
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(r *repository.Repository, id backend.ID) error {
debug.Log("Checker.checkPack", "checking pack %v", id.Str())
rd, err := r.Backend().Get(backend.Data, id.String())
if err != nil {
return err
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
return err
}
err = rd.Close()
if err != nil {
return err
}
unpacker, err := pack.NewUnpacker(r.Key(), bytes.NewReader(buf))
if err != nil {
return err
}
var errs []error
for i, blob := range unpacker.Entries {
debug.Log("Checker.checkPack", " check blob %d: %v", i, blob.ID.Str())
plainBuf := make([]byte, blob.Length)
plainBuf, err = crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
if err != nil {
debug.Log("Checker.checkPack", " error decrypting blob %v: %v", blob.ID.Str(), err)
errs = append(errs, fmt.Errorf("blob %v: %v", i, err))
continue
}
hash := backend.Hash(plainBuf)
if !hash.Equal(blob.ID) {
debug.Log("Checker.checkPack", " ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
errs = append(errs, fmt.Errorf("ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))
continue
}
}
if len(errs) > 0 {
return fmt.Errorf("pack %v contains %v errors: %v", id.Str(), len(errs), errs)
}
return nil
}
func fsckFile(global CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint64, error) {
debug.Log("restic.fsckFile", "checking file %v", IDs)
var bytes uint64
for _, id := range IDs {
debug.Log("restic.fsck", " checking data blob %v\n", id)
// test if blob is in the index
packID, tpe, _, length, err := repo.Index().Lookup(id)
if err != nil {
return 0, fmt.Errorf("storage for blob %v (%v) not found", id, tpe)
}
bytes += uint64(length - crypto.Extension)
debug.Log("restic.fsck", " blob found in pack %v\n", packID)
if global.CheckData {
// load content
_, err := repo.LoadBlob(pack.Data, id)
if err != nil {
return 0, err
}
} else {
// test if pack for data blob is there
ok, err := repo.Backend().Test(backend.Data, packID.String())
if err != nil {
return 0, err
}
if !ok {
return 0, fmt.Errorf("data blob %v not found", id)
}
}
// if orphan check is active, record storage id
if global.o_data != nil {
debug.Log("restic.fsck", " recording blob %v as used\n", id)
global.o_data.Insert(id)
}
}
return bytes, nil
}