// download downloads the given file from remote storage using the get function,
// adding hashes to the path if consistent snapshots are in use
func (c *Client) download(file string, get remoteGetFunc, hashes data.Hashes) (io.ReadCloser, int64, error) {
if c.consistentSnapshot {
// try each hashed path in turn, and either return the contents,
// try the next one if a 404 is returned, or return an error
for _, path := range util.HashedPaths(file, hashes) {
r, size, err := get(path)
if err != nil {
if IsNotFound(err) {
continue
}
return nil, 0, err
}
return r, size, nil
}
return nil, 0, ErrNotFound{file}
} else {
return get(file)
}
}
func (t *tmpDir) assertHashedFilesNotExist(path string, hashes data.Hashes) {
t.c.Assert(len(hashes) > 0, Equals, true)
for _, path := range util.HashedPaths(path, hashes) {
t.assertNotExist(path)
}
}
func (f *fileSystemStore) Commit(meta map[string]json.RawMessage, consistentSnapshot bool, hashes map[string]data.Hashes) error {
isTarget := func(path string) bool {
return strings.HasPrefix(path, "targets/")
}
shouldCopyHashed := func(path string) bool {
return consistentSnapshot && path != "timestamp.json"
}
shouldCopyUnhashed := func(path string) bool {
return !consistentSnapshot || !isTarget(path)
}
copyToRepo := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() || !info.Mode().IsRegular() {
return nil
}
rel, err := filepath.Rel(f.stagedDir(), path)
if err != nil {
return err
}
var paths []string
if shouldCopyHashed(rel) {
paths = append(paths, util.HashedPaths(rel, hashes[rel])...)
}
if shouldCopyUnhashed(rel) {
paths = append(paths, rel)
}
var files []io.Writer
for _, path := range paths {
file, err := f.createRepoFile(path)
if err != nil {
return err
}
defer file.Close()
files = append(files, file)
}
staged, err := os.Open(path)
if err != nil {
return err
}
defer staged.Close()
if _, err = io.Copy(io.MultiWriter(files...), staged); err != nil {
return err
}
return nil
}
needsRemoval := func(path string) bool {
if consistentSnapshot {
// strip out the hash
name := strings.SplitN(filepath.Base(path), ".", 2)
if name[1] == "" {
return false
}
path = filepath.Join(filepath.Dir(path), name[1])
}
_, ok := hashes[path]
return !ok
}
removeFile := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(f.repoDir(), path)
if err != nil {
return err
}
if !info.IsDir() && isTarget(rel) && needsRemoval(rel) {
if err := os.Remove(path); err != nil {
// TODO: log / handle error
}
// TODO: remove empty directory
}
return nil
}
if err := filepath.Walk(f.stagedDir(), copyToRepo); err != nil {
return err
}
if err := filepath.Walk(f.repoDir(), removeFile); err != nil {
return err
}
return f.Clean()
}