// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are s3://host/bucketname/prefix and
// s3:host:bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
switch {
case strings.HasPrefix(s, "s3:http"):
// assume that a URL has been specified, parse it and
// use the host as the endpoint and the path as the
// bucket name and prefix
url, err := url.Parse(s[3:])
if err != nil {
return nil, errors.Wrap(err, "url.Parse")
}
if url.Path == "" {
return nil, errors.New("s3: bucket name not found")
}
path := strings.SplitN(url.Path[1:], "/", 2)
return createConfig(url.Host, path, url.Scheme == "http")
case strings.HasPrefix(s, "s3://"):
s = s[5:]
case strings.HasPrefix(s, "s3:"):
s = s[3:]
default:
return nil, errors.New("s3: invalid format")
}
// use the first entry of the path as the endpoint and the
// remainder as bucket name and prefix
path := strings.SplitN(s, "/", 3)
return createConfig(path[0], path[1:], false)
}
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
if h.Type == "" {
return errors.New("type is empty")
}
switch h.Type {
case DataFile:
case KeyFile:
case LockFile:
case SnapshotFile:
case IndexFile:
case ConfigFile:
default:
return errors.Errorf("invalid Type %q", h.Type)
}
if h.Type == ConfigFile {
return nil
}
if h.Name == "" {
return errors.New("invalid Name")
}
return nil
}
// forgetfulBackend returns a backend that forgets everything.
func forgetfulBackend() restic.Backend {
be := &mock.Backend{}
be.TestFn = func(t restic.FileType, name string) (bool, error) {
return false, nil
}
be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
return 0, errors.New("not found")
}
be.SaveFn = func(h restic.Handle, p []byte) error {
return nil
}
be.StatFn = func(h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, errors.New("not found")
}
be.RemoveFn = func(t restic.FileType, name string) error {
return nil
}
be.ListFn = func(t restic.FileType, done <-chan struct{}) <-chan string {
ch := make(chan string)
close(ch)
return ch
}
be.DeleteFn = func() error {
return nil
}
return be
}
// readHeader reads the header at the end of rd. size is the length of the
// whole data accessible in rd.
func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
hl, err := readHeaderLength(rd, size)
if err != nil {
return nil, err
}
if int64(hl) > size-int64(binary.Size(hl)) {
return nil, errors.New("header is larger than file")
}
if int64(hl) > maxHeaderSize {
return nil, errors.New("header is larger than maxHeaderSize")
}
buf := make([]byte, int(hl))
n, err := rd.ReadAt(buf, size-int64(hl)-int64(binary.Size(hl)))
if err != nil {
return nil, errors.Wrap(err, "ReadAt")
}
if n != len(buf) {
return nil, errors.New("not enough bytes read")
}
return buf, nil
}
func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
debug.Log("checking tree %v", id.Str())
var blobs []restic.ID
for _, node := range tree.Nodes {
switch node.Type {
case "file":
if node.Content == nil {
errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)})
}
for b, blobID := range node.Content {
if blobID.IsNull() {
errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)})
continue
}
blobs = append(blobs, blobID)
}
case "dir":
if node.Subtree == nil {
errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)})
continue
}
if node.Subtree.IsNull() {
errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q subtree id is null", node.Name)})
continue
}
case "symlink", "socket", "chardev", "dev", "fifo":
// nothing to check
default:
errs = append(errs, Error{TreeID: id, Err: errors.Errorf("node %q with invalid type %q", node.Name, node.Type)})
}
if node.Name == "" {
errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")})
}
}
for _, blobID := range blobs {
c.blobRefs.Lock()
c.blobRefs.M[blobID]++
debug.Log("blob %v refcount %d", blobID.Str(), c.blobRefs.M[blobID])
c.blobRefs.Unlock()
if !c.blobs.Has(blobID) {
debug.Log("tree %v references blob %v which isn't contained in index", id.Str(), blobID.Str())
errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")})
}
}
return errs
}
// writeToTempfile saves p into a tempfile in tempdir.
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
tmpfile, err := ioutil.TempFile(tempdir, "temp-")
if err != nil {
return "", errors.Wrap(err, "TempFile")
}
n, err := tmpfile.Write(p)
if err != nil {
return "", errors.Wrap(err, "Write")
}
if n != len(p) {
return "", errors.New("not all bytes writen")
}
if err = tmpfile.Sync(); err != nil {
return "", errors.Wrap(err, "Syncn")
}
err = tmpfile.Close()
if err != nil {
return "", errors.Wrap(err, "Close")
}
return tmpfile.Name(), nil
}
func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) {
debug.Log("worker start")
defer debug.Log("worker done")
defer wg.Done()
for id := range inChan {
ok, err := repo.Backend().Test(restic.DataFile, id.String())
if err != nil {
err = PackError{ID: id, Err: err}
} else {
if !ok {
err = PackError{ID: id, Err: errors.New("does not exist")}
}
}
if err != nil {
debug.Log("error checking for pack %s: %v", id.Str(), err)
select {
case <-done:
return
case errChan <- err:
}
continue
}
debug.Log("pack %s exists", id.Str())
}
}
// Delete calls backend.Delete() if implemented, and returns an error
// otherwise.
func (r *Repository) Delete() error {
if b, ok := r.be.(restic.Deleter); ok {
return b.Delete()
}
return errors.New("Delete() called for backend that does not implement this method")
}
// Encrypt encrypts and authenticates the plaintext and saves the result in
// ciphertext.
func (r *Repository) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
if r.key == nil {
return nil, errors.New("key for repository not set")
}
return crypto.Encrypt(r.key, ciphertext, plaintext)
}
// decrypt authenticates and decrypts ciphertext and stores the result in
// plaintext.
func (r *Repository) decryptTo(plaintext, ciphertext []byte) (int, error) {
if r.key == nil {
return 0, errors.New("key for repository not set")
}
return crypto.Decrypt(r.key, plaintext, ciphertext)
}
// Delete all data.
func (m *Backend) Delete() error {
if m.DeleteFn == nil {
return errors.New("not implemented")
}
return m.DeleteFn()
}
// Test for the existence of a specific item.
func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
if m.TestFn == nil {
return false, errors.New("not implemented")
}
return m.TestFn(t, name)
}
// Remove data from the backend.
func (m *Backend) Remove(t restic.FileType, name string) error {
if m.RemoveFn == nil {
return errors.New("not implemented")
}
return m.RemoveFn(t, name)
}
// Stat an object in the backend.
func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) {
if m.StatFn == nil {
return restic.FileInfo{}, errors.New("not implemented")
}
return m.StatFn(h)
}
// Save data in the backend.
func (m *Backend) Save(h restic.Handle, p []byte) error {
if m.SaveFn == nil {
return errors.New("not implemented")
}
return m.SaveFn(h, p)
}
// Load loads data from the backend.
func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
if m.LoadFn == nil {
return 0, errors.New("not implemented")
}
return m.LoadFn(h, p, off)
}
// Remove removes the blob with the given name and type.
func (b *restBackend) Remove(t restic.FileType, name string) error {
h := restic.Handle{Type: t, Name: name}
if err := h.Valid(); err != nil {
return err
}
req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
if err != nil {
return errors.Wrap(err, "http.NewRequest")
}
<-b.connChan
resp, err := b.client.Do(req)
b.connChan <- struct{}{}
if err != nil {
return errors.Wrap(err, "client.Do")
}
if resp.StatusCode != 200 {
return errors.New("blob not removed")
}
io.Copy(ioutil.Discard, resp.Body)
return resp.Body.Close()
}
// ParseConfig parses a local backend config.
func ParseConfig(cfg string) (interface{}, error) {
if !strings.HasPrefix(cfg, "local:") {
return nil, errors.New(`invalid format, prefix "local" not found`)
}
return cfg[6:], nil
}
// Stat returns information about a blob.
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil {
return restic.FileInfo{}, err
}
<-b.connChan
resp, err := b.client.Head(restPath(b.url, h))
b.connChan <- struct{}{}
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
}
io.Copy(ioutil.Discard, resp.Body)
if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close")
}
if resp.StatusCode != 200 {
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}
if resp.ContentLength < 0 {
return restic.FileInfo{}, errors.New("negative content length")
}
bi := restic.FileInfo{
Size: resp.ContentLength,
}
return bi, nil
}
// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created. `dir` must
// be delimited by forward slashes ("/"), which is required by sftp.
func Create(dir string, program string, args ...string) (*SFTP, error) {
debug.Log("%v %v", program, args)
sftp, err := startClient(program, args...)
if err != nil {
return nil, err
}
// test if config file already exists
_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
if err == nil {
return nil, errors.New("config file already exists")
}
// create paths for data, refs and temp blobs
for _, d := range paths(dir) {
err = sftp.mkdirAll(d, backend.Modes.Dir)
if err != nil {
return nil, err
}
}
err = sftp.Close()
if err != nil {
return nil, errors.Wrap(err, "Close")
}
// open backend
return Open(dir, program, args...)
}
// Save stores data in the backend at the handle.
func (be s3) Save(h restic.Handle, p []byte) (err error) {
if err := h.Valid(); err != nil {
return err
}
debug.Log("%v with %d bytes", h, len(p))
path := be.s3path(h.Type, h.Name)
// Check key does not already exist
_, err = be.client.StatObject(be.bucketname, path)
if err == nil {
debug.Log("%v already exists", h)
return errors.New("key already exists")
}
<-be.connChan
defer func() {
be.connChan <- struct{}{}
}()
debug.Log("PutObject(%v, %v, %v, %v)",
be.bucketname, path, int64(len(p)), "binary/octet-stream")
n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream")
debug.Log("%v -> %v bytes, err %#v", path, n, err)
return errors.Wrap(err, "client.PutObject")
}
// OpenKey tries do decrypt the key specified by name with the given password.
func OpenKey(s *Repository, name string, password string) (*Key, error) {
k, err := LoadKey(s, name)
if err != nil {
debug.Log("LoadKey(%v) returned error %v", name[:12], err)
return nil, err
}
// check KDF
if k.KDF != "scrypt" {
return nil, errors.New("only supported KDF is scrypt()")
}
// derive user key
params := crypto.KDFParams{
N: k.N,
R: k.R,
P: k.P,
}
k.user, err = crypto.KDF(params, k.Salt, password)
if err != nil {
return nil, errors.Wrap(err, "crypto.KDF")
}
// decrypt master keys
buf := make([]byte, len(k.Data))
n, err := crypto.Decrypt(k.user, buf, k.Data)
if err != nil {
return nil, err
}
buf = buf[:n]
// restore json
k.master = &crypto.Key{}
err = json.Unmarshal(buf, k.master)
if err != nil {
debug.Log("Unmarshal() returned error %v", err)
return nil, errors.Wrap(err, "Unmarshal")
}
k.name = name
if !k.Valid() {
return nil, errors.New("Invalid key for repository")
}
return k, nil
}
// DeviceID extracts the device ID from an os.FileInfo object by casting it
// to syscall.Stat_t
func DeviceID(fi os.FileInfo) (deviceID uint64, err error) {
if st, ok := fi.Sys().(*syscall.Stat_t); ok {
// st.Dev is uint32 on Darwin and uint64 on Linux. Just cast
// everything to uint64.
return uint64(st.Dev), nil
}
return 0, errors.New("Could not cast to syscall.Stat_t")
}
// SetID sets the ID the index has been written to. This requires that
// Finalize() has been called before, otherwise an error is returned.
func (idx *Index) SetID(id restic.ID) error {
idx.m.Lock()
defer idx.m.Unlock()
if !idx.final {
return errors.New("indexs is not final")
}
if !idx.id.IsNull() {
return errors.New("ID already set")
}
debug.Log("ID set to %v", id.Str())
idx.id = id
return nil
}
func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
buf, ok := m.blobs[id]
if !ok {
return 0, errors.New("blob not found")
}
return uint(len(buf)), nil
}
// ParseConfig parses the string s and extracts the sftp config. The
// supported configuration formats are sftp://user@host/directory
// (with an optional port sftp://user@host:port/directory) and
// sftp:user@host:directory. The directory will be path Cleaned and
// can be an absolute path if it starts with a '/'
// (e.g. sftp://user@host//absolute and sftp:user@host:/absolute).
func ParseConfig(s string) (interface{}, error) {
var user, host, dir string
switch {
case strings.HasPrefix(s, "sftp://"):
// parse the "sftp://user@host/path" url format
url, err := url.Parse(s)
if err != nil {
return nil, errors.Wrap(err, "url.Parse")
}
if url.User != nil {
user = url.User.Username()
}
host = url.Host
dir = url.Path
if dir == "" {
return nil, errors.Errorf("invalid backend %q, no directory specified", s)
}
dir = dir[1:]
case strings.HasPrefix(s, "sftp:"):
// parse the sftp:user@host:path format, which means we'll get
// "user@host:path" in s
s = s[5:]
// split user@host and path at the colon
data := strings.SplitN(s, ":", 2)
if len(data) < 2 {
return nil, errors.New("sftp: invalid format, hostname or path not found")
}
host = data[0]
dir = data[1]
// split user and host at the "@"
data = strings.SplitN(host, "@", 2)
if len(data) == 2 {
user = data[0]
host = data[1]
}
default:
return nil, errors.New(`invalid format, does not start with "sftp:"`)
}
return Config{
User: user,
Host: host,
Dir: path.Clean(dir),
}, nil
}
func init() {
if TestS3Server == "" {
SkipMessage = "s3 test server not available"
return
}
url, err := url.Parse(TestS3Server)
if err != nil {
fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
return
}
cfg := s3.Config{
Endpoint: url.Host,
Bucket: "restictestbucket",
KeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
}
if url.Scheme == "http" {
cfg.UseHTTP = true
}
test.CreateFn = func() (restic.Backend, error) {
be, err := s3.Open(cfg)
if err != nil {
return nil, err
}
exists, err := be.Test(restic.ConfigFile, "")
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
}
test.OpenFn = func() (restic.Backend, error) {
return s3.Open(cfg)
}
// test.CleanupFn = func() error {
// if tempBackendDir == "" {
// return nil
// }
// fmt.Printf("removing test backend at %v\n", tempBackendDir)
// err := os.RemoveAll(tempBackendDir)
// tempBackendDir = ""
// return err
// }
}
// ID returns the ID of the index, if available. If the index is not yet
// finalized, an error is returned.
func (idx *Index) ID() (restic.ID, error) {
idx.m.Lock()
defer idx.m.Unlock()
if !idx.final {
return restic.ID{}, errors.New("index not finalized")
}
return idx.id, nil
}
// MarshalJSON encodes the BlobType into JSON.
func (t BlobType) MarshalJSON() ([]byte, error) {
switch t {
case DataBlob:
return []byte(`"data"`), nil
case TreeBlob:
return []byte(`"tree"`), nil
}
return nil, errors.New("unknown blob type")
}
// AddToSupersedes adds the ids to the list of indexes superseded by this
// index. If the index has already been finalized, an error is returned.
func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
idx.m.Lock()
defer idx.m.Unlock()
if idx.final {
return errors.New("index already finalized")
}
idx.supersedes = append(idx.supersedes, ids...)
return nil
}