Example #1
0
// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are s3://host/bucketname/prefix and
// s3:host:bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
	switch {
	case strings.HasPrefix(s, "s3:http"):
		// assume that a URL has been specified, parse it and
		// use the host as the endpoint and the path as the
		// bucket name and prefix
		url, err := url.Parse(s[3:])
		if err != nil {
			return nil, errors.Wrap(err, "url.Parse")
		}

		if url.Path == "" {
			return nil, errors.New("s3: bucket name not found")
		}

		path := strings.SplitN(url.Path[1:], "/", 2)
		return createConfig(url.Host, path, url.Scheme == "http")
	case strings.HasPrefix(s, "s3://"):
		s = s[5:]
	case strings.HasPrefix(s, "s3:"):
		s = s[3:]
	default:
		return nil, errors.New("s3: invalid format")
	}
	// use the first entry of the path as the endpoint and the
	// remainder as bucket name and prefix
	path := strings.SplitN(s, "/", 3)
	return createConfig(path[0], path[1:], false)
}
Example #2
0
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
	if h.Type == "" {
		return errors.New("type is empty")
	}

	switch h.Type {
	case DataFile:
	case KeyFile:
	case LockFile:
	case SnapshotFile:
	case IndexFile:
	case ConfigFile:
	default:
		return errors.Errorf("invalid Type %q", h.Type)
	}

	if h.Type == ConfigFile {
		return nil
	}

	if h.Name == "" {
		return errors.New("invalid Name")
	}

	return nil
}
// forgetfulBackend returns a backend that forgets everything.
func forgetfulBackend() restic.Backend {
	be := &mock.Backend{}

	be.TestFn = func(t restic.FileType, name string) (bool, error) {
		return false, nil
	}

	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
		return 0, errors.New("not found")
	}

	be.SaveFn = func(h restic.Handle, p []byte) error {
		return nil
	}

	be.StatFn = func(h restic.Handle) (restic.FileInfo, error) {
		return restic.FileInfo{}, errors.New("not found")
	}

	be.RemoveFn = func(t restic.FileType, name string) error {
		return nil
	}

	be.ListFn = func(t restic.FileType, done <-chan struct{}) <-chan string {
		ch := make(chan string)
		close(ch)
		return ch
	}

	be.DeleteFn = func() error {
		return nil
	}

	return be
}
Example #4
0
// readHeader reads the header at the end of rd. size is the length of the
// whole data accessible in rd.
func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
	hl, err := readHeaderLength(rd, size)
	if err != nil {
		return nil, err
	}

	if int64(hl) > size-int64(binary.Size(hl)) {
		return nil, errors.New("header is larger than file")
	}

	if int64(hl) > maxHeaderSize {
		return nil, errors.New("header is larger than maxHeaderSize")
	}

	buf := make([]byte, int(hl))
	n, err := rd.ReadAt(buf, size-int64(hl)-int64(binary.Size(hl)))
	if err != nil {
		return nil, errors.Wrap(err, "ReadAt")
	}

	if n != len(buf) {
		return nil, errors.New("not enough bytes read")
	}

	return buf, nil
}
Example #5
0
func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
	debug.Log("checking tree %v", id.Str())

	var blobs []restic.ID

	for _, node := range tree.Nodes {
		switch node.Type {
		case "file":
			if node.Content == nil {
				errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)})
			}

			for b, blobID := range node.Content {
				if blobID.IsNull() {
					errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)})
					continue
				}
				blobs = append(blobs, blobID)
			}
		case "dir":
			if node.Subtree == nil {
				errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)})
				continue
			}

			if node.Subtree.IsNull() {
				errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q subtree id is null", node.Name)})
				continue
			}

		case "symlink", "socket", "chardev", "dev", "fifo":
			// nothing to check

		default:
			errs = append(errs, Error{TreeID: id, Err: errors.Errorf("node %q with invalid type %q", node.Name, node.Type)})
		}

		if node.Name == "" {
			errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")})
		}
	}

	for _, blobID := range blobs {
		c.blobRefs.Lock()
		c.blobRefs.M[blobID]++
		debug.Log("blob %v refcount %d", blobID.Str(), c.blobRefs.M[blobID])
		c.blobRefs.Unlock()

		if !c.blobs.Has(blobID) {
			debug.Log("tree %v references blob %v which isn't contained in index", id.Str(), blobID.Str())

			errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")})
		}
	}

	return errs
}
Example #6
0
// writeToTempfile saves p into a tempfile in tempdir.
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
	if err != nil {
		return "", errors.Wrap(err, "TempFile")
	}

	n, err := tmpfile.Write(p)
	if err != nil {
		return "", errors.Wrap(err, "Write")
	}

	if n != len(p) {
		return "", errors.New("not all bytes writen")
	}

	if err = tmpfile.Sync(); err != nil {
		return "", errors.Wrap(err, "Syncn")
	}

	err = tmpfile.Close()
	if err != nil {
		return "", errors.Wrap(err, "Close")
	}

	return tmpfile.Name(), nil
}
Example #7
0
func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) {
	debug.Log("worker start")
	defer debug.Log("worker done")

	defer wg.Done()

	for id := range inChan {
		ok, err := repo.Backend().Test(restic.DataFile, id.String())
		if err != nil {
			err = PackError{ID: id, Err: err}
		} else {
			if !ok {
				err = PackError{ID: id, Err: errors.New("does not exist")}
			}
		}

		if err != nil {
			debug.Log("error checking for pack %s: %v", id.Str(), err)
			select {
			case <-done:
				return
			case errChan <- err:
			}

			continue
		}

		debug.Log("pack %s exists", id.Str())
	}
}
Example #8
0
// Delete calls backend.Delete() if implemented, and returns an error
// otherwise.
func (r *Repository) Delete() error {
	if b, ok := r.be.(restic.Deleter); ok {
		return b.Delete()
	}

	return errors.New("Delete() called for backend that does not implement this method")
}
Example #9
0
// Encrypt encrypts and authenticates the plaintext and saves the result in
// ciphertext.
func (r *Repository) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
	if r.key == nil {
		return nil, errors.New("key for repository not set")
	}

	return crypto.Encrypt(r.key, ciphertext, plaintext)
}
Example #10
0
// decrypt authenticates and decrypts ciphertext and stores the result in
// plaintext.
func (r *Repository) decryptTo(plaintext, ciphertext []byte) (int, error) {
	if r.key == nil {
		return 0, errors.New("key for repository not set")
	}

	return crypto.Decrypt(r.key, plaintext, ciphertext)
}
Example #11
0
// Delete all data.
func (m *Backend) Delete() error {
	if m.DeleteFn == nil {
		return errors.New("not implemented")
	}

	return m.DeleteFn()
}
Example #12
0
// Test for the existence of a specific item.
func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
	if m.TestFn == nil {
		return false, errors.New("not implemented")
	}

	return m.TestFn(t, name)
}
Example #13
0
// Remove data from the backend.
func (m *Backend) Remove(t restic.FileType, name string) error {
	if m.RemoveFn == nil {
		return errors.New("not implemented")
	}

	return m.RemoveFn(t, name)
}
Example #14
0
// Stat an object in the backend.
func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) {
	if m.StatFn == nil {
		return restic.FileInfo{}, errors.New("not implemented")
	}

	return m.StatFn(h)
}
Example #15
0
// Save data in the backend.
func (m *Backend) Save(h restic.Handle, p []byte) error {
	if m.SaveFn == nil {
		return errors.New("not implemented")
	}

	return m.SaveFn(h, p)
}
Example #16
0
// Load loads data from the backend.
func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
	if m.LoadFn == nil {
		return 0, errors.New("not implemented")
	}

	return m.LoadFn(h, p, off)
}
Example #17
0
// Remove removes the blob with the given name and type.
func (b *restBackend) Remove(t restic.FileType, name string) error {
	h := restic.Handle{Type: t, Name: name}
	if err := h.Valid(); err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
	if err != nil {
		return errors.Wrap(err, "http.NewRequest")
	}
	<-b.connChan
	resp, err := b.client.Do(req)
	b.connChan <- struct{}{}

	if err != nil {
		return errors.Wrap(err, "client.Do")
	}

	if resp.StatusCode != 200 {
		return errors.New("blob not removed")
	}

	io.Copy(ioutil.Discard, resp.Body)
	return resp.Body.Close()
}
Example #18
0
// ParseConfig parses a local backend config.
func ParseConfig(cfg string) (interface{}, error) {
	if !strings.HasPrefix(cfg, "local:") {
		return nil, errors.New(`invalid format, prefix "local" not found`)
	}

	return cfg[6:], nil
}
Example #19
0
// Stat returns information about a blob.
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
	if err := h.Valid(); err != nil {
		return restic.FileInfo{}, err
	}

	<-b.connChan
	resp, err := b.client.Head(restPath(b.url, h))
	b.connChan <- struct{}{}
	if err != nil {
		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
	}

	io.Copy(ioutil.Discard, resp.Body)
	if err = resp.Body.Close(); err != nil {
		return restic.FileInfo{}, errors.Wrap(err, "Close")
	}

	if resp.StatusCode != 200 {
		return restic.FileInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
	}

	if resp.ContentLength < 0 {
		return restic.FileInfo{}, errors.New("negative content length")
	}

	bi := restic.FileInfo{
		Size: resp.ContentLength,
	}

	return bi, nil
}
Example #20
0
// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created. `dir` must
// be delimited by forward slashes ("/"), which is required by sftp.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	debug.Log("%v %v", program, args)
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range paths(dir) {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.Close()
	if err != nil {
		return nil, errors.Wrap(err, "Close")
	}

	// open backend
	return Open(dir, program, args...)
}
Example #21
0
// Save stores data in the backend at the handle.
func (be s3) Save(h restic.Handle, p []byte) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	debug.Log("%v with %d bytes", h, len(p))

	path := be.s3path(h.Type, h.Name)

	// Check key does not already exist
	_, err = be.client.StatObject(be.bucketname, path)
	if err == nil {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	<-be.connChan
	defer func() {
		be.connChan <- struct{}{}
	}()

	debug.Log("PutObject(%v, %v, %v, %v)",
		be.bucketname, path, int64(len(p)), "binary/octet-stream")
	n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream")
	debug.Log("%v -> %v bytes, err %#v", path, n, err)

	return errors.Wrap(err, "client.PutObject")
}
Example #22
0
// OpenKey tries do decrypt the key specified by name with the given password.
func OpenKey(s *Repository, name string, password string) (*Key, error) {
	k, err := LoadKey(s, name)
	if err != nil {
		debug.Log("LoadKey(%v) returned error %v", name[:12], err)
		return nil, err
	}

	// check KDF
	if k.KDF != "scrypt" {
		return nil, errors.New("only supported KDF is scrypt()")
	}

	// derive user key
	params := crypto.KDFParams{
		N: k.N,
		R: k.R,
		P: k.P,
	}
	k.user, err = crypto.KDF(params, k.Salt, password)
	if err != nil {
		return nil, errors.Wrap(err, "crypto.KDF")
	}

	// decrypt master keys
	buf := make([]byte, len(k.Data))
	n, err := crypto.Decrypt(k.user, buf, k.Data)
	if err != nil {
		return nil, err
	}
	buf = buf[:n]

	// restore json
	k.master = &crypto.Key{}
	err = json.Unmarshal(buf, k.master)
	if err != nil {
		debug.Log("Unmarshal() returned error %v", err)
		return nil, errors.Wrap(err, "Unmarshal")
	}
	k.name = name

	if !k.Valid() {
		return nil, errors.New("Invalid key for repository")
	}

	return k, nil
}
Example #23
0
// DeviceID extracts the device ID from an os.FileInfo object by casting it
// to syscall.Stat_t
func DeviceID(fi os.FileInfo) (deviceID uint64, err error) {
	if st, ok := fi.Sys().(*syscall.Stat_t); ok {
		// st.Dev is uint32 on Darwin and uint64 on Linux. Just cast
		// everything to uint64.
		return uint64(st.Dev), nil
	}
	return 0, errors.New("Could not cast to syscall.Stat_t")
}
Example #24
0
// SetID sets the ID the index has been written to. This requires that
// Finalize() has been called before, otherwise an error is returned.
func (idx *Index) SetID(id restic.ID) error {
	idx.m.Lock()
	defer idx.m.Unlock()

	if !idx.final {
		return errors.New("indexs is not final")
	}

	if !idx.id.IsNull() {
		return errors.New("ID already set")
	}

	debug.Log("ID set to %v", id.Str())
	idx.id = id

	return nil
}
Example #25
0
func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
	buf, ok := m.blobs[id]
	if !ok {
		return 0, errors.New("blob not found")
	}

	return uint(len(buf)), nil
}
Example #26
0
// ParseConfig parses the string s and extracts the sftp config. The
// supported configuration formats are sftp://user@host/directory
// (with an optional port sftp://user@host:port/directory) and
// sftp:user@host:directory.  The directory will be path Cleaned and
// can be an absolute path if it starts with a '/'
// (e.g. sftp://user@host//absolute and sftp:user@host:/absolute).
func ParseConfig(s string) (interface{}, error) {
	var user, host, dir string
	switch {
	case strings.HasPrefix(s, "sftp://"):
		// parse the "sftp://user@host/path" url format
		url, err := url.Parse(s)
		if err != nil {
			return nil, errors.Wrap(err, "url.Parse")
		}
		if url.User != nil {
			user = url.User.Username()
		}
		host = url.Host
		dir = url.Path
		if dir == "" {
			return nil, errors.Errorf("invalid backend %q, no directory specified", s)
		}

		dir = dir[1:]
	case strings.HasPrefix(s, "sftp:"):
		// parse the sftp:user@host:path format, which means we'll get
		// "user@host:path" in s
		s = s[5:]
		// split user@host and path at the colon
		data := strings.SplitN(s, ":", 2)
		if len(data) < 2 {
			return nil, errors.New("sftp: invalid format, hostname or path not found")
		}
		host = data[0]
		dir = data[1]
		// split user and host at the "@"
		data = strings.SplitN(host, "@", 2)
		if len(data) == 2 {
			user = data[0]
			host = data[1]
		}
	default:
		return nil, errors.New(`invalid format, does not start with "sftp:"`)
	}
	return Config{
		User: user,
		Host: host,
		Dir:  path.Clean(dir),
	}, nil
}
Example #27
0
func init() {
	if TestS3Server == "" {
		SkipMessage = "s3 test server not available"
		return
	}

	url, err := url.Parse(TestS3Server)
	if err != nil {
		fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
		return
	}

	cfg := s3.Config{
		Endpoint: url.Host,
		Bucket:   "restictestbucket",
		KeyID:    os.Getenv("AWS_ACCESS_KEY_ID"),
		Secret:   os.Getenv("AWS_SECRET_ACCESS_KEY"),
	}

	if url.Scheme == "http" {
		cfg.UseHTTP = true
	}

	test.CreateFn = func() (restic.Backend, error) {
		be, err := s3.Open(cfg)
		if err != nil {
			return nil, err
		}

		exists, err := be.Test(restic.ConfigFile, "")
		if err != nil {
			return nil, err
		}

		if exists {
			return nil, errors.New("config already exists")
		}

		return be, nil
	}

	test.OpenFn = func() (restic.Backend, error) {
		return s3.Open(cfg)
	}

	// test.CleanupFn = func() error {
	// 	if tempBackendDir == "" {
	// 		return nil
	// 	}

	// 	fmt.Printf("removing test backend at %v\n", tempBackendDir)
	// 	err := os.RemoveAll(tempBackendDir)
	// 	tempBackendDir = ""
	// 	return err
	// }
}
Example #28
0
// ID returns the ID of the index, if available. If the index is not yet
// finalized, an error is returned.
func (idx *Index) ID() (restic.ID, error) {
	idx.m.Lock()
	defer idx.m.Unlock()

	if !idx.final {
		return restic.ID{}, errors.New("index not finalized")
	}

	return idx.id, nil
}
Example #29
0
// MarshalJSON encodes the BlobType into JSON.
func (t BlobType) MarshalJSON() ([]byte, error) {
	switch t {
	case DataBlob:
		return []byte(`"data"`), nil
	case TreeBlob:
		return []byte(`"tree"`), nil
	}

	return nil, errors.New("unknown blob type")
}
Example #30
0
// AddToSupersedes adds the ids to the list of indexes superseded by this
// index. If the index has already been finalized, an error is returned.
func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
	idx.m.Lock()
	defer idx.m.Unlock()

	if idx.final {
		return errors.New("index already finalized")
	}

	idx.supersedes = append(idx.supersedes, ids...)
	return nil
}