func TestTarOutputCompat(t *testing.T) {
Convey("Output should produce a tar recognizable to gnu tar", t,
testutil.Requires(
testutil.RequiresRoot,
testutil.WithTmpdir(func() {
for _, fixture := range filefixture.All {
Convey(fmt.Sprintf("- Fixture %q", fixture.Name), func() {
// create fixture
fixture.Create("./data")
// scan it
transmat := New("./workdir")
transmat.Scan(
Kind,
"./data",
[]integrity.SiloURI{
integrity.SiloURI("file://output.tar"),
},
)
// sanity check that there's a file.
So("./output.tar", testutil.ShouldBeFile, os.FileMode(0))
// now exec tar, and check that it doesn't barf outright.
// this is not well isolated from the host; consider improving that a todo.
os.Mkdir("./untar", 0755)
tarProc := gosh.Gosh(
"tar",
"-xf", "./output.tar",
"-C", "./untar",
gosh.NullIO,
).RunAndReport()
So(tarProc.GetExitCode(), ShouldEqual, 0)
// should look roughly the same again even bounced through
// some third-party tar implementation, one would hope.
rescan := filefixture.Scan("./untar")
comparisonLevel := filefixture.CompareDefaults &^ filefixture.CompareSubsecond
So(rescan.Describe(comparisonLevel), ShouldEqual, fixture.Describe(comparisonLevel))
})
}
}),
),
)
}
/*
Arenas produced by Dir Transmats may be relocated by simple `mv`.
*/
func (t *TarExecTransmat) Materialize(
kind integrity.TransmatKind,
dataHash integrity.CommitID,
siloURIs []integrity.SiloURI,
options ...integrity.MaterializerConfigurer,
) integrity.Arena {
var arena dirArena
try.Do(func() {
// Basic validation and config
if !(kind == Kind || kind == "exec-tar") {
panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
}
// Ping silos
if len(siloURIs) < 1 {
panic(integrity.ConfigError.New("Materialization requires at least one data source!"))
// Note that it's possible a caching layer will satisfy things even without data sources...
// but if that was going to happen, it already would have by now.
}
// Our policy is to take the first path that exists.
// This lets you specify a series of potential locations, and if one is unavailable we'll just take the next.
var siloURI integrity.SiloURI
for _, givenURI := range siloURIs {
// TODO still assuming all local paths and not doing real uri parsing
localPath := string(givenURI)
_, err := os.Stat(localPath)
if os.IsNotExist(err) {
// TODO it'd be awfully lovely if we could log the attempt somewhere
continue
}
siloURI = givenURI
break
}
if siloURI == "" {
panic(integrity.WarehouseConnectionError.New("No warehouses were available!"))
}
// Open the input stream; preparing decompression as necessary
file, err := os.OpenFile(string(siloURI), os.O_RDONLY, 0755)
if err != nil {
panic(integrity.WarehouseConnectionError.New("Unable to read file: %s", err))
}
file.Close() // just checking, so we can (try to) give a more pleasant error than tar barf
// Create staging arena to produce data into.
arena.path, err = ioutil.TempDir(t.workPath, "")
if err != nil {
panic(integrity.TransmatError.New("Unable to create arena: %s", err))
}
// exec tar.
// in case of a zero (a.k.a. success) exit, this returns silently.
// in case of a non-zero exit, this panics; the panic will include the output.
gosh.Gosh(
"tar",
"-xf", string(siloURI),
"-C", arena.Path(),
gosh.NullIO,
).RunAndReport()
// note: indeed, we never check the hash field. this is *not* a compliant implementation of an input.
}).Catch(integrity.Error, func(err *errors.Error) {
panic(err)
}).CatchAll(func(err error) {
panic(integrity.UnknownError.Wrap(err))
}).Done()
return arena
}
func (t TarExecTransmat) Scan(
kind integrity.TransmatKind,
subjectPath string,
siloURIs []integrity.SiloURI,
options ...integrity.MaterializerConfigurer,
) integrity.CommitID {
try.Do(func() {
// Basic validation and config
if kind != Kind {
panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
}
// If scan area doesn't exist, bail immediately.
// No need to even start dialing warehouses if we've got nothing for em.
_, err := os.Stat(subjectPath)
if err != nil {
if os.IsNotExist(err) {
return // empty commitID
} else {
panic(err)
}
}
// Parse save locations.
// (Most transmats do... significantly smarter things than this backwater.)
var localPath string
if len(siloURIs) == 0 {
localPath = "/dev/null"
} else if len(siloURIs) == 1 {
// TODO still assuming all local paths and not doing real uri parsing
localPath = string(siloURIs[0])
err := os.MkdirAll(filepath.Dir(localPath), 0755)
if err != nil {
panic(integrity.WarehouseConnectionError.New("Unable to write file: %s", err))
}
file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
panic(integrity.WarehouseConnectionError.New("Unable to write file: %s", err))
}
file.Close() // just checking, so we can (try to) give a more pleasant error than tar barf
} else {
panic(integrity.ConfigError.New("%s transmat only supports shipping to 1 warehouse", Kind))
}
// exec tar.
// in case of a zero (a.k.a. success) exit, this returns silently.
// in case of a non-zero exit, this panics; the panic will include the output.
gosh.Gosh(
"tar",
"-cf", localPath,
"--xform", "s,"+strings.TrimLeft(subjectPath, "/")+",.,",
subjectPath,
gosh.NullIO,
).RunAndReport()
}).Catch(integrity.Error, func(err *errors.Error) {
panic(err)
}).CatchAll(func(err error) {
panic(integrity.UnknownError.Wrap(err))
}).Done()
return ""
}
func New(workPath string) integrity.Transmat {
err := os.MkdirAll(workPath, 0755)
if err != nil {
panic(integrity.TransmatError.New("Unable to set up workspace: %s", err))
}
workPath, err = filepath.Abs(workPath)
if err != nil {
panic(integrity.TransmatError.New("Unable to set up workspace: %s", err))
}
return &GitTransmat{workPath}
}
var git gosh.Command = gosh.Gosh(
"git",
gosh.NullIO,
gosh.Opts{Env: map[string]string{
"GIT_CONFIG_NOSYSTEM": "true",
"HOME": "/dev/null",
}},
)
/*
Git transmats plonk down the contents of one commit (or tree) as a filesystem.
A fileset materialized by git does *not* include the `.git` dir by default,
since those files are not themselves part of what's described by the hash.
Git effectively "filters" out several attributes -- permissions are only loosely
respected (execution only), file timestamps are undefined, uid/gid bits
are not tracked, xattrs are not tracked, etc. If you desired defined values,
*you must still configure materialization to use a filter* (particularly for
file timestamps, since they will otherwise be allowed to vary from one
func TestTarInputCompat(t *testing.T) {
projPath, _ := os.Getwd()
projPath = filepath.Dir(filepath.Dir(filepath.Dir(projPath)))
Convey("Unpacking tars should match exec untar", t,
testutil.Requires(testutil.RequiresRoot, testutil.WithTmpdir(func() {
checkEquivalence := func(hash, filename string, paveBase bool) {
transmat := New("./workdir/tar")
// apply it; hope it doesn't blow up
arena := transmat.Materialize(
integrity.TransmatKind("tar"),
integrity.CommitID(hash),
[]integrity.SiloURI{
integrity.SiloURI("file://" + filename),
},
)
defer arena.Teardown()
// do a native untar; since we don't have an upfront fixture
// for this thing, we'll compare the two as filesystems.
// this is not well isolated from the host; consider improving that a todo.
os.Mkdir("./untar", 0755)
tarProc := gosh.Gosh(
"tar",
"-xf", filename,
"-C", "./untar",
gosh.NullIO,
).RunAndReport()
So(tarProc.GetExitCode(), ShouldEqual, 0)
// native untar may or may not have an opinion about the base dir, depending on how it was formed.
// but our scans do, so, if the `paveBase` flag was set to warn us that the tar was missing an "./" entry, flatten that here.
if paveBase {
So(fspatch.LUtimesNano("./untar", def.Epochwhen, def.Epochwhen), ShouldBeNil)
}
// scan and compare
scan1 := filefixture.Scan(arena.Path())
scan2 := filefixture.Scan("./untar")
// boy, that's entertaining though: gnu tar does all the same stuff,
// except it doesn't honor our nanosecond timings.
// also exclude bodies because they're *big*.
comparisonLevel := filefixture.CompareDefaults &^ filefixture.CompareSubsecond &^ filefixture.CompareBody
So(scan1.Describe(comparisonLevel), ShouldEqual, scan2.Describe(comparisonLevel))
}
Convey("Given a fixture tarball complete with base dir", func() {
checkEquivalence(
"BX0jm4jRNCg1KMbZfv4zp7ZaShx9SUXKaDrO-Xy6mWIoWOCFP5VnDHDDR3nU4PrR",
filepath.Join(projPath, "data/fixture/tar_withBase.tgz"),
false,
)
})
Convey("Given a fixture tarball lacking base dir", func() {
checkEquivalence(
"ZdV3xhCGWeJmsfeHpDF4nF9stwvdskYwcepKMcOf7a2ziax1YGjQvGTJjRWFkvG1",
filepath.Join(projPath, "data/fixture/tar_sansBase.tgz"),
true,
)
})
Convey("Given a fixture tarball containing ubuntu",
testutil.Requires(testutil.RequiresLongRun, func() {
checkEquivalence(
ubuntuTarballHash,
filepath.Join(projPath, "assets/ubuntu.tar.gz"),
true,
)
}),
)
})),
)
Convey("Bouncing unusual tars should match hash", t,
// where really all "unusual" means is "valid tar, but not from our own cannonical output".
testutil.Requires(
testutil.RequiresRoot,
testutil.WithTmpdir(func() {
checkBounce := func(hash, filename string) {
transmat := New("./workdir/tar")
// apply it; hope it doesn't blow up
arena := transmat.Materialize(
integrity.TransmatKind("tar"),
integrity.CommitID(hash),
[]integrity.SiloURI{
integrity.SiloURI("file://" + filename),
},
)
defer arena.Teardown()
// scan and compare
commitID := transmat.Scan(integrity.TransmatKind("tar"), arena.Path(), nil)
// debug: gosh.Sh("tar", "--utc", "-xOvf", filename)
So(commitID, ShouldEqual, integrity.CommitID(hash))
}
Convey("Given a fixture tarball complete with base dir", func() {
checkBounce(
"BX0jm4jRNCg1KMbZfv4zp7ZaShx9SUXKaDrO-Xy6mWIoWOCFP5VnDHDDR3nU4PrR",
filepath.Join(projPath, "data/fixture/tar_withBase.tgz"),
)
})
Convey("Given a fixture tarball lacking base dir", func() {
checkBounce(
"ZdV3xhCGWeJmsfeHpDF4nF9stwvdskYwcepKMcOf7a2ziax1YGjQvGTJjRWFkvG1",
filepath.Join(projPath, "data/fixture/tar_sansBase.tgz"),
)
})
// this won't fly until we support hardlinks; the original asset uses them.
// Convey("Given a fixture tarball containing ubuntu",
// testutil.Requires(testutil.RequiresLongRun, func() {
// checkBounce(
// ubuntuTarballHash,
// filepath.Join(projPath, "assets/ubuntu.tar.gz"),
// )
// }),
// )
}),
),
)
}