// ListPack returns the list of blobs saved in the pack id and the length of
// the file as stored in the backend.
func (r *Repository) ListPack(id restic.ID) ([]restic.Blob, int64, error) {
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
blobInfo, err := r.Backend().Stat(h)
if err != nil {
return nil, 0, err
}
blobs, err := pack.List(r.Key(), restic.ReaderAt(r.Backend(), h), blobInfo.Size)
if err != nil {
return nil, 0, err
}
return blobs, blobInfo.Size, nil
}
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(r restic.Repository, id restic.ID) error {
debug.Log("checking pack %v", id.Str())
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
buf, err := backend.LoadAll(r.Backend(), h, nil)
if err != nil {
return err
}
hash := restic.Hash(buf)
if !hash.Equal(id) {
debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
}
blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
if err != nil {
return err
}
var errs []error
for i, blob := range blobs {
debug.Log(" check blob %d: %v", i, blob.ID.Str())
plainBuf := make([]byte, blob.Length)
n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
if err != nil {
debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue
}
plainBuf = plainBuf[:n]
hash := restic.Hash(plainBuf)
if !hash.Equal(blob.ID) {
debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))
continue
}
}
if len(errs) > 0 {
return errors.Errorf("pack %v contains %v errors: %v", id.Str(), len(errs), errs)
}
return nil
}
func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSize uint) {
written := 0
for _, buf := range bufs {
written += len(buf.data)
}
// header length
written += binary.Size(uint32(0))
// header
written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
// header crypto
written += crypto.Extension
// check length
Equals(t, uint(written), packSize)
// read and parse it again
entries, err := pack.List(k, rd, int64(packSize))
OK(t, err)
Equals(t, len(entries), len(bufs))
var buf []byte
for i, b := range bufs {
e := entries[i]
Equals(t, b.id, e.ID)
if len(buf) < int(e.Length) {
buf = make([]byte, int(e.Length))
}
buf = buf[:int(e.Length)]
n, err := rd.ReadAt(buf, int64(e.Offset))
OK(t, err)
buf = buf[:n]
Assert(t, bytes.Equal(b.data, buf),
"data for blob %v doesn't match", i)
}
}
// Repack takes a list of packs together with a list of blobs contained in
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
buf := make([]byte, 0, maxPackSize)
for packID := range packs {
// load the complete pack
h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
if errors.Cause(err) == io.ErrUnexpectedEOF {
err = nil
buf = buf[:l]
}
if err != nil {
return err
}
debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))
blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
if err != nil {
return err
}
debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
var plaintext []byte
for _, entry := range blobs {
h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
if !keepBlobs.Has(h) {
continue
}
debug.Log(" process blob %v", h)
ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
plaintext = plaintext[:len(plaintext)]
if len(plaintext) < len(ciphertext) {
plaintext = make([]byte, len(ciphertext))
}
debug.Log(" ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))
n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
if err != nil {
return err
}
plaintext = plaintext[:n]
_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
if err != nil {
return err
}
debug.Log(" saved blob %v", entry.ID.Str())
keepBlobs.Delete(h)
}
}
if err := repo.Flush(); err != nil {
return err
}
for packID := range packs {
err := repo.Backend().Remove(restic.DataFile, packID.String())
if err != nil {
debug.Log("error removing pack %v: %v", packID.Str(), err)
return err
}
debug.Log("removed pack %v", packID.Str())
}
return nil
}
func printPacks(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
name := job.Data.(string)
h := restic.Handle{Type: restic.DataFile, Name: name}
blobInfo, err := repo.Backend().Stat(h)
if err != nil {
return nil, err
}
blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
if err != nil {
return nil, err
}
return blobs, nil
}
jobCh := make(chan worker.Job)
resCh := make(chan worker.Job)
wp := worker.New(dumpPackWorkers, f, jobCh, resCh)
go func() {
for name := range repo.Backend().List(restic.DataFile, done) {
jobCh <- worker.Job{Data: name}
}
close(jobCh)
}()
for job := range resCh {
name := job.Data.(string)
if job.Error != nil {
fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
continue
}
entries := job.Result.([]restic.Blob)
p := Pack{
Name: name,
Blobs: make([]Blob, len(entries)),
}
for i, blob := range entries {
p.Blobs[i] = Blob{
Type: blob.Type,
Length: blob.Length,
ID: blob.ID,
Offset: blob.Offset,
}
}
prettyPrintJSON(os.Stdout, p)
}
wp.Wait()
return nil
}