/*
Much like `os.MkdirAll`, but standardizes values (mtime, etc) to match the
given metadata as it goes.
Existing directories are not modified; the metadata is applied only to
newly created directories.
Note that "exising directories are not modified" should be read as "...intentionally".
As usual, a filesystem with mtime and atime behaviors enabled may change
those attributes on the top level dir. If this function created any new dirs,
it attempt to modify the mtime of the parent to replace its original value; but
note that this is inherently a best-effort scenario and subject to races.
*/
func MkdirAllWithAttribs(path string, hdr Metadata) error {
stack, topMTime, err := mkdirAll(path, hdr)
if err != nil {
return err
}
if stack == nil {
return nil
}
for i := len(stack) - 1; i >= 0; i-- {
if err := fspatch.LUtimesNano(stack[i], hdr.AccessTime, hdr.ModTime); err != nil {
return err
}
}
top := filepath.Dir(stack[0])
if top != "." {
if err := fspatch.LUtimesNano(top, def.Epochwhen, topMTime); err != nil {
// gave up and reset atime to epoch. sue me. atimes are ridiculous.
return err
}
}
return nil
}
func TestTarInputCompat(t *testing.T) {
projPath, _ := os.Getwd()
projPath = filepath.Dir(filepath.Dir(filepath.Dir(projPath)))
Convey("Unpacking tars should match exec untar", t,
testutil.Requires(testutil.RequiresRoot, testutil.WithTmpdir(func() {
checkEquivalence := func(hash, filename string, paveBase bool) {
transmat := New("./workdir/tar")
// apply it; hope it doesn't blow up
arena := transmat.Materialize(
integrity.TransmatKind("tar"),
integrity.CommitID(hash),
[]integrity.SiloURI{
integrity.SiloURI("file://" + filename),
},
)
defer arena.Teardown()
// do a native untar; since we don't have an upfront fixture
// for this thing, we'll compare the two as filesystems.
// this is not well isolated from the host; consider improving that a todo.
os.Mkdir("./untar", 0755)
tarProc := gosh.Gosh(
"tar",
"-xf", filename,
"-C", "./untar",
gosh.NullIO,
).RunAndReport()
So(tarProc.GetExitCode(), ShouldEqual, 0)
// native untar may or may not have an opinion about the base dir, depending on how it was formed.
// but our scans do, so, if the `paveBase` flag was set to warn us that the tar was missing an "./" entry, flatten that here.
if paveBase {
So(fspatch.LUtimesNano("./untar", def.Epochwhen, def.Epochwhen), ShouldBeNil)
}
// scan and compare
scan1 := filefixture.Scan(arena.Path())
scan2 := filefixture.Scan("./untar")
// boy, that's entertaining though: gnu tar does all the same stuff,
// except it doesn't honor our nanosecond timings.
// also exclude bodies because they're *big*.
comparisonLevel := filefixture.CompareDefaults &^ filefixture.CompareSubsecond &^ filefixture.CompareBody
So(scan1.Describe(comparisonLevel), ShouldEqual, scan2.Describe(comparisonLevel))
}
Convey("Given a fixture tarball complete with base dir", func() {
checkEquivalence(
"BX0jm4jRNCg1KMbZfv4zp7ZaShx9SUXKaDrO-Xy6mWIoWOCFP5VnDHDDR3nU4PrR",
filepath.Join(projPath, "data/fixture/tar_withBase.tgz"),
false,
)
})
Convey("Given a fixture tarball lacking base dir", func() {
checkEquivalence(
"ZdV3xhCGWeJmsfeHpDF4nF9stwvdskYwcepKMcOf7a2ziax1YGjQvGTJjRWFkvG1",
filepath.Join(projPath, "data/fixture/tar_sansBase.tgz"),
true,
)
})
Convey("Given a fixture tarball containing ubuntu",
testutil.Requires(testutil.RequiresLongRun, func() {
checkEquivalence(
ubuntuTarballHash,
filepath.Join(projPath, "assets/ubuntu.tar.gz"),
true,
)
}),
)
})),
)
Convey("Bouncing unusual tars should match hash", t,
// where really all "unusual" means is "valid tar, but not from our own cannonical output".
testutil.Requires(
testutil.RequiresRoot,
testutil.WithTmpdir(func() {
checkBounce := func(hash, filename string) {
transmat := New("./workdir/tar")
// apply it; hope it doesn't blow up
arena := transmat.Materialize(
integrity.TransmatKind("tar"),
integrity.CommitID(hash),
[]integrity.SiloURI{
integrity.SiloURI("file://" + filename),
},
)
defer arena.Teardown()
// scan and compare
commitID := transmat.Scan(integrity.TransmatKind("tar"), arena.Path(), nil)
// debug: gosh.Sh("tar", "--utc", "-xOvf", filename)
So(commitID, ShouldEqual, integrity.CommitID(hash))
}
Convey("Given a fixture tarball complete with base dir", func() {
checkBounce(
"BX0jm4jRNCg1KMbZfv4zp7ZaShx9SUXKaDrO-Xy6mWIoWOCFP5VnDHDDR3nU4PrR",
filepath.Join(projPath, "data/fixture/tar_withBase.tgz"),
)
})
Convey("Given a fixture tarball lacking base dir", func() {
checkBounce(
"ZdV3xhCGWeJmsfeHpDF4nF9stwvdskYwcepKMcOf7a2ziax1YGjQvGTJjRWFkvG1",
filepath.Join(projPath, "data/fixture/tar_sansBase.tgz"),
)
})
// this won't fly until we support hardlinks; the original asset uses them.
// Convey("Given a fixture tarball containing ubuntu",
// testutil.Requires(testutil.RequiresLongRun, func() {
// checkBounce(
// ubuntuTarballHash,
// filepath.Join(projPath, "assets/ubuntu.tar.gz"),
// )
// }),
// )
}),
),
)
}
/*
Places a file on the filesystem.
Replicates all attributes described in the metadata.
The path is the join of `destBasePath` and `hdr.Name`.
`hdr.Name` should be the full relative path of the file;
if it has an absolute prefix, that's quietly ignored, and it's treated as relative anyway.
`hdr.Name` should match `filepath.Clean` output, except that it must always
use the unix directory separator.
No changes are allowed to occur outside of `destBasePath`.
Hardlinks may not point outside of the base path.
Symlinks may *point* at paths outside of the base path (because you
may be about to chroot into this, in which case absolute link paths
make perfect sense), and invalid symlinks are acceptable -- however
symlinks may *not* be traversed during any part of `hdr.Name`; this is
considered malformed input and will result in a BreakoutError.
`destBasePath` MUST be absolute. Isolation checks assume this, and
have undefined operation if this requirement is not met.
Please note that like all filesystem operations within a lightyear of
symlinks, all validations are best-effort, but are only capable of
correctness in the absense of concurrent modifications inside `destBasePath`.
Device files *will* be created, with their maj/min numbers.
This may be considered a security concern; you should whitelist inputs
if using this to provision a sandbox.
*/
func PlaceFile(destBasePath string, hdr Metadata, body io.Reader) {
destPath := filepath.Join(destBasePath, hdr.Name)
mode := hdr.FileMode()
// First, no part of the path may be a symlink.
// We *could* create an application-level jailing effect as we walk this,
// but that's just complicated enough to be dangerous, and also still
// results in a world where results would vary depending on order of `PlaceFile` calls.
// So! Traversing symlinks during placement is fiat unacceptable.
parts := strings.Split(hdr.Name, "/")
for i := range parts {
target, err := os.Readlink(filepath.Join(append([]string{destBasePath}, parts[:i]...)...))
if err != nil {
if os.IsNotExist(err) {
continue
} else if err.(*os.PathError).Err == syscall.EINVAL {
continue
} else {
ioError(err)
}
}
panic(BreakoutError.New("placefile: refusing to traverse symlink at %q->%q while placing %q", filepath.Join(parts[:i]...), target, hdr.Name))
}
switch hdr.Typeflag {
case tar.TypeDir:
if hdr.Name == "./" {
// for the base dir only:
// the dir may exist; we'll just chown+chmod+chtime it.
// there is no race-free path through this btw, unless you know of a way to lstat and mkdir in the same syscall.
if fi, err := os.Lstat(destPath); err == nil && fi.IsDir() {
break
}
}
if err := os.Mkdir(destPath, mode); err != nil {
ioError(err)
}
case tar.TypeReg, tar.TypeRegA:
file, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, mode)
if err != nil {
ioError(err)
}
if _, err := io.Copy(file, body); err != nil {
file.Close()
ioError(err)
}
file.Close()
case tar.TypeSymlink:
// linkname can be anything you want. it can be invalid, it can be absolute, whatever.
// the consumer had better know how to jail this filesystem before using;
// other PlaceFile calls know enough to refuse to traverse this.
if err := os.Symlink(hdr.Linkname, destPath); err != nil {
ioError(err)
}
case tar.TypeLink:
targetPath := filepath.Join(destBasePath, hdr.Linkname)
if !strings.HasPrefix(targetPath, destBasePath) {
panic(BreakoutError.New("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
}
if err := os.Link(targetPath, destPath); err != nil {
ioError(err)
}
case tar.TypeBlock:
mode := uint32(hdr.Mode&07777) | syscall.S_IFBLK
if err := syscall.Mknod(destPath, mode, int(fspatch.Mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
ioError(err)
}
case tar.TypeChar:
mode := uint32(hdr.Mode&07777) | syscall.S_IFCHR
if err := syscall.Mknod(destPath, mode, int(fspatch.Mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
ioError(err)
}
case tar.TypeFifo:
if err := syscall.Mkfifo(destPath, uint32(hdr.Mode&07777)); err != nil {
ioError(err)
}
default:
panic(errors.NotImplementedError.New("The tennants of filesystems have changed! We're not prepared for this file mode %q", hdr.Typeflag))
}
if err := os.Lchown(destPath, hdr.Uid, hdr.Gid); err != nil {
ioError(err)
}
for key, value := range hdr.Xattrs {
if err := fspatch.Lsetxattr(destPath, key, []byte(value), 0); err != nil {
ioError(err)
}
}
if hdr.Typeflag != tar.TypeSymlink {
// do this for everything not a symlink, since there's no such thing as `lchmod` on linux -.-
if err := os.Chmod(destPath, mode); err != nil {
ioError(err)
}
}
if err := fspatch.LUtimesNano(destPath, hdr.AccessTime, hdr.ModTime); err != nil {
ioError(err)
}
}
// Exposed only because you're probably doing your own trees somehow, and it's
// necessary to cover your tracks by forcing times on dirs after all children are done.
// Symmetric params and error handling to `PlaceFile` for your convenience.
func PlaceDirTime(destBasePath string, hdr Metadata) {
destPath := filepath.Join(destBasePath, hdr.Name)
if err := fspatch.LUtimesNano(destPath, hdr.AccessTime, hdr.ModTime); err != nil {
ioError(err)
}
}