Example #1
0
// ListPack returns the list of blobs saved in the pack id and the length of
// the file as stored in the backend.
func (r *Repository) ListPack(id restic.ID) ([]restic.Blob, int64, error) {
	h := restic.Handle{Type: restic.DataFile, Name: id.String()}

	blobInfo, err := r.Backend().Stat(h)
	if err != nil {
		return nil, 0, err
	}

	blobs, err := pack.List(r.Key(), restic.ReaderAt(r.Backend(), h), blobInfo.Size)
	if err != nil {
		return nil, 0, err
	}

	return blobs, blobInfo.Size, nil
}
Example #2
0
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(r restic.Repository, id restic.ID) error {
	debug.Log("checking pack %v", id.Str())
	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
	buf, err := backend.LoadAll(r.Backend(), h, nil)
	if err != nil {
		return err
	}

	hash := restic.Hash(buf)
	if !hash.Equal(id) {
		debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
		return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
	}

	blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
	if err != nil {
		return err
	}

	var errs []error
	for i, blob := range blobs {
		debug.Log("  check blob %d: %v", i, blob.ID.Str())

		plainBuf := make([]byte, blob.Length)
		n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
		if err != nil {
			debug.Log("  error decrypting blob %v: %v", blob.ID.Str(), err)
			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
			continue
		}
		plainBuf = plainBuf[:n]

		hash := restic.Hash(plainBuf)
		if !hash.Equal(blob.ID) {
			debug.Log("  Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
			errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))
			continue
		}
	}

	if len(errs) > 0 {
		return errors.Errorf("pack %v contains %v errors: %v", id.Str(), len(errs), errs)
	}

	return nil
}
Example #3
0
func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSize uint) {
	written := 0
	for _, buf := range bufs {
		written += len(buf.data)
	}
	// header length
	written += binary.Size(uint32(0))
	// header
	written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
	// header crypto
	written += crypto.Extension

	// check length
	Equals(t, uint(written), packSize)

	// read and parse it again
	entries, err := pack.List(k, rd, int64(packSize))
	OK(t, err)
	Equals(t, len(entries), len(bufs))

	var buf []byte
	for i, b := range bufs {
		e := entries[i]
		Equals(t, b.id, e.ID)

		if len(buf) < int(e.Length) {
			buf = make([]byte, int(e.Length))
		}
		buf = buf[:int(e.Length)]
		n, err := rd.ReadAt(buf, int64(e.Offset))
		OK(t, err)
		buf = buf[:n]

		Assert(t, bytes.Equal(b.data, buf),
			"data for blob %v doesn't match", i)
	}
}
Example #4
0
// Repack takes a list of packs together with a list of blobs contained in
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))

	buf := make([]byte, 0, maxPackSize)
	for packID := range packs {
		// load the complete pack
		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}

		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
		if errors.Cause(err) == io.ErrUnexpectedEOF {
			err = nil
			buf = buf[:l]
		}

		if err != nil {
			return err
		}

		debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))

		blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
		if err != nil {
			return err
		}

		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
		var plaintext []byte
		for _, entry := range blobs {
			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
			if !keepBlobs.Has(h) {
				continue
			}

			debug.Log("  process blob %v", h)

			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
			plaintext = plaintext[:len(plaintext)]
			if len(plaintext) < len(ciphertext) {
				plaintext = make([]byte, len(ciphertext))
			}

			debug.Log("  ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))

			n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
			if err != nil {
				return err
			}
			plaintext = plaintext[:n]

			_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
			if err != nil {
				return err
			}

			debug.Log("  saved blob %v", entry.ID.Str())

			keepBlobs.Delete(h)
		}
	}

	if err := repo.Flush(); err != nil {
		return err
	}

	for packID := range packs {
		err := repo.Backend().Remove(restic.DataFile, packID.String())
		if err != nil {
			debug.Log("error removing pack %v: %v", packID.Str(), err)
			return err
		}
		debug.Log("removed pack %v", packID.Str())
	}

	return nil
}
Example #5
0
func printPacks(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
		name := job.Data.(string)

		h := restic.Handle{Type: restic.DataFile, Name: name}

		blobInfo, err := repo.Backend().Stat(h)
		if err != nil {
			return nil, err
		}

		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
		if err != nil {
			return nil, err
		}

		return blobs, nil
	}

	jobCh := make(chan worker.Job)
	resCh := make(chan worker.Job)
	wp := worker.New(dumpPackWorkers, f, jobCh, resCh)

	go func() {
		for name := range repo.Backend().List(restic.DataFile, done) {
			jobCh <- worker.Job{Data: name}
		}
		close(jobCh)
	}()

	for job := range resCh {
		name := job.Data.(string)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
			continue
		}

		entries := job.Result.([]restic.Blob)
		p := Pack{
			Name:  name,
			Blobs: make([]Blob, len(entries)),
		}
		for i, blob := range entries {
			p.Blobs[i] = Blob{
				Type:   blob.Type,
				Length: blob.Length,
				ID:     blob.ID,
				Offset: blob.Offset,
			}
		}

		prettyPrintJSON(os.Stdout, p)
	}

	wp.Wait()

	return nil
}