func TwerkCommandPattern(stdin io.Reader, stdout, stderr io.Writer) cli.Command {
return cli.Command{
Name: "twerk",
Usage: "Run one-time-use interactive (thus nonrepeatable!) command. All the defaults are filled in for you. Great for experimentation.",
Action: func(ctx *cli.Context) {
executor := executordispatch.Get("chroot")
formula := def.Formula{
Inputs: []def.Input{{
Type: "tar",
MountPath: "/",
Hash: "uJRF46th6rYHt0zt_n3fcDuBfGFVPS6lzRZla5hv6iDoh5DVVzxUTMMzENfPoboL",
Warehouses: []string{
"http+ca://repeatr.s3.amazonaws.com/assets/",
},
}},
Action: def.Action{
Entrypoint: []string{"bash", "-c", "echo hallo ; pwd ; ls -la ; bash"},
},
}
// TODO bonus points if you eventually can get the default mode to have no setuid binaries, in addition to making a spare user and dropping privs immediately.
fmt.Fprintln(ctx.App.Writer, "launchin")
job := executor.Start(formula, def.JobID(guid.New()), stdin, ctx.App.Writer)
go io.Copy(stdout, job.Outputs().Reader(1))
go io.Copy(stderr, job.Outputs().Reader(2))
result := job.Wait()
if result.Error != nil {
fmt.Fprintf(ctx.App.Writer, "error: %s\n", result.Error)
} else {
fmt.Fprintf(ctx.App.Writer, "done; exit code %d\n", result.ExitCode)
}
},
}
}
func soExpectSuccessAndOutput(execEng executor.Executor, formula def.Formula, output string) {
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldEqual, output)
}
func CheckFilesystemContainment(execEng executor.Executor) {
Convey("SPEC: Launching with multiple inputs should work", func() {
formula := getBaseFormula()
Convey("Launch should succeed", func() {
filefixture.Beta.Create("./fixture/beta")
formula.Inputs = append(formula.Inputs, (def.Input{
Name: "2-input-test",
Type: "dir",
Hash: filefixture.Beta_Hash,
Warehouses: []string{"./fixture/beta"},
MountPath: "/data/test",
}))
formula.Action = def.Action{
Entrypoint: []string{"/bin/true"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
Convey("Commands inside the job should be able to see the mounted files", FailureContinues, func() {
formula.Action = def.Action{
Entrypoint: []string{"ls", "/data/test"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldEqual, "1\n2\n3\n")
})
})
})
}
func CheckEnvBehavior(execEng executor.Executor) {
// NOTE the chroot executor currently uses `def.ValidateAll` which effectively *does not permit you to have an empty $PATH var*.
// we may decide to change that later -- but there's a correctness versus convenience battle there, and for now, we're going with convenience.
Convey("SPEC: Env vars should be contained", func() {
formula := getBaseFormula()
formula.Action = def.Action{
Entrypoint: []string{"env"},
}
Convey("Env from the parent should not be inherited", func() {
os.Setenv("REPEATR_TEST_KEY_1", "test value")
defer os.Unsetenv("REPEATR_TEST_KEY_1") // using unique strings per test anyway, because this is too scary
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(strings.Contains("REPEATR_TEST_KEY_1", string(msg)), ShouldBeFalse)
})
Convey("Env specified with the job should be applied", func() {
formula.Action.Env = make(map[string]string)
formula.Action.Env["REPEATR_TEST_KEY_2"] = "test value"
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldContainSubstring, "REPEATR_TEST_KEY_2=test value")
})
})
}
func CheckPwdBehavior(execEng executor.Executor) {
Convey("SPEC: Working directory should be contained", func() {
formula := getBaseFormula()
Convey("The default pwd should be the root", func() {
// This test exists because of a peculularly terrifying fact about chroots:
// If you spawn a new process with the chroot without setting its current working dir,
// the cwd is still *whatever it inherits*. And even if the process can't reach there
// starting from "/", it can still *walk deeper from that cwd*.
formula.Action = def.Action{
//Entrypoint: []string{"find", "-maxdepth", "5"}, // if you goofed, during test runs this will show you the executor's workspace!
//Entrypoint: []string{"bash", "-c", "find -maxdepth 5 ; echo --- ; echo \"$PWD\" ; cd \"$(echo \"$PWD\" | sed 's/^(unreachable)//')\" ; ls"}, // this demo's that you can't actually cd back to it, though.
Entrypoint: []string{"pwd"},
}
soExpectSuccessAndOutput(execEng, formula,
"/\n",
)
})
Convey("Setting another cwd should work", func() {
formula.Action = def.Action{
Cwd: "/usr",
Entrypoint: []string{"pwd"},
}
soExpectSuccessAndOutput(execEng, formula,
"/usr\n",
)
})
Convey("Setting a nonexistent cwd should fail to launch", FailureContinues, func() {
formula.Action = def.Action{
Cwd: "/does/not/exist/by/any/means",
Entrypoint: []string{"pwd"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldNotBeNil)
So(job.Wait().Error, testutil.ShouldBeErrorClass, executor.TaskExecError)
So(job.Wait().ExitCode, ShouldEqual, -1)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldEqual, "")
})
})
}
func makeWriteController(warehouseCoords integrity.SiloURI) StreamingWarehouseWriteController {
u, err := url.Parse(string(warehouseCoords))
if err != nil {
panic(integrity.ConfigError.New("failed to parse URI: %s", err))
}
controller := &fileWarehouseWriteController{
pathPrefix: u.Path,
}
switch u.Scheme {
case "file+ca":
controller.ctntAddr = true
fallthrough
case "file":
// Pick a random upload path
controller.pathPrefix = filepath.Join(u.Host, controller.pathPrefix) // file uris don't have hosts
if controller.ctntAddr {
controller.tmpPath = filepath.Join(controller.pathPrefix, ".tmp.upload."+guid.New())
} else {
controller.tmpPath = filepath.Join(path.Dir(controller.pathPrefix), ".tmp.upload."+path.Base(controller.pathPrefix)+"."+guid.New())
}
// Check if warehouse path exists.
// Warehouse is expected to exist already; transmats
// should *not* create one whimsically, that's someone else's responsibility.
warehouseBasePath := filepath.Dir(controller.tmpPath)
if _, err := os.Stat(warehouseBasePath); err != nil {
panic(integrity.WarehouseConnectionError.New("Warehouse unavailable: %q %s", warehouseBasePath, err))
}
// Open file to shovel data into
file, err := os.OpenFile(controller.tmpPath, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
panic(integrity.WarehouseConnectionError.New("Unable to write %q: %s", controller.tmpPath, err))
}
controller.stream = file
return controller
case "http+ca":
fallthrough
case "http":
fallthrough
case "https+ca":
fallthrough
case "https":
panic(integrity.ConfigError.New("http transports are only supported for read-only use"))
case "":
panic(integrity.ConfigError.New("missing scheme in warehouse URI; need a prefix, e.g. \"file://\" or \"http://\""))
default:
panic(integrity.ConfigError.New("unsupported scheme in warehouse URI: %q", u.Scheme))
}
}
func (s *Scheduler) Schedule(f def.Formula) (def.JobID, <-chan def.Job) {
id := def.JobID(guid.New())
h := &hold{
id: id,
forumla: f,
response: make(chan def.Job),
}
// Non-blocking send, will panic if scheduler queue is full
select {
case s.queue <- h:
default:
panic(scheduler.QueueFullError)
}
return id, h.response
}
func TestCoreCompliance(t *testing.T) {
if _, err := s3gof3r.EnvKeys(); err != nil {
t.Skipf("skipping s3 output tests; no s3 credentials loaded (err: %s)", err)
}
// group all effects of this test run under one "dir" for human reader sanity and cleanup in extremis.
testRunGuid := guid.New()
Convey("Spec Compliance: S3 Transmat", t, testutil.WithTmpdir(func() {
// scanning
tests.CheckScanWithoutMutation(Kind, New)
tests.CheckScanProducesConsistentHash(Kind, New)
tests.CheckScanProducesDistinctHashes(Kind, New)
tests.CheckScanEmptyIsCalm(Kind, New)
tests.CheckScanWithFilters(Kind, New)
// round-trip
tests.CheckRoundTrip(Kind, New, "s3://repeatr-test/test-"+testRunGuid+"/rt/obj.tar", "literal path")
tests.CheckRoundTrip(Kind, New, "s3+splay://repeatr-test/test-"+testRunGuid+"/rt-splay/heap/", "content addressible path")
}))
}
func (t S3Transmat) Scan(
kind integrity.TransmatKind,
subjectPath string,
siloURIs []integrity.SiloURI,
options ...integrity.MaterializerConfigurer,
) integrity.CommitID {
var commitID integrity.CommitID
try.Do(func() {
// Basic validation and config
config := integrity.EvaluateConfig(options...)
if kind != Kind {
panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
}
// If scan area doesn't exist, bail immediately.
// No need to even start dialing warehouses if we've got nothing for em.
_, err := os.Stat(subjectPath)
if err != nil {
if os.IsNotExist(err) {
return // empty commitID
} else {
panic(err)
}
}
// load keys from env
// TODO someday URIs should grow smart enough to control this in a more general fashion -- but for now, host ENV is actually pretty feasible and plays easily with others.
keys, err := s3gof3r.EnvKeys()
if err != nil {
panic(S3CredentialsMissingError.Wrap(err))
}
// Parse URI; Find warehouses; Open output streams for writing.
// Since these are all behaving as just one `io.Writer` stream, this could maybe be factored out.
// Error handling is currently "anything -> panic". This should probably be more resilient. (That might need another refactor so we have an upload call per remote.)
// TODO : both this and the tar code that has a similar single stream idea should use an interface
// And that interface should have a concept of mv so we can make atomic commits.
// I'm not doing multiple URIs here until we get that, because the io.Writer interface just
// doesn't cut it like it did for tars (and really, it's ignoring a major issue to use it there, too).
// ...F**k it, we're gonna do it
controllers := make([]*s3warehousePut, 0)
writers := make([]io.Writer, 0) // this is dumb, but we end up making one of these to satisfy the type conversation for MultiWriter anyway
for _, givenURI := range siloURIs {
u, err := url.Parse(string(givenURI))
if err != nil {
panic(integrity.ConfigError.New("failed to parse URI: %s", err))
}
controller := &s3warehousePut{}
controller.bucketName = u.Host
controller.pathPrefix = u.Path
var ctntAddr bool
switch u.Scheme {
case "s3":
ctntAddr = false
case "s3+splay":
ctntAddr = true
default:
panic(integrity.ConfigError.New("unrecognized scheme: %q", u.Scheme))
}
// dial it and initialize writer to s3!
// if the URI indicated splay behavior, first stream data to {$bucketName}:{dirname($storePath)}/.tmp.upload.{basename($storePath)}.{random()};
// this allows us to start uploading before the final hash is determined and relocate it later.
// for direct paths, upload into place, because aws already manages atomicity at that scale (and they don't have a rename or copy operation that's free, because uh...? no time to implement it since 2006, apparently).
controller.keys = keys
if ctntAddr {
controller.tmpPath = path.Join(
path.Dir(controller.pathPrefix),
".tmp.upload."+path.Base(controller.pathPrefix)+"."+guid.New(),
)
controller.stream = makeS3writer(controller.bucketName, controller.tmpPath, keys)
} else {
controller.stream = makeS3writer(controller.bucketName, controller.pathPrefix, keys)
}
controllers = append(controllers, controller)
writers = append(writers, controller.stream)
}
stream := io.MultiWriter(writers...)
if len(writers) < 1 {
stream = ioutil.Discard
}
// walk, fwrite, hash
commitID = integrity.CommitID(tartrans.Save(stream, subjectPath, config.FilterSet, hasherFactory))
// commit
for _, controller := range controllers {
controller.Commit(string(commitID))
}
}).Catch(integrity.Error, func(err *errors.Error) {
panic(err)
}).CatchAll(func(err error) {
panic(integrity.UnknownError.Wrap(err))
}).Done()
return commitID
}
/*
Check the basics of exec:
- Does it work at all?
- Can we see error codes?
- Can we stream stdout?
- If there's no rootfs, does it panic?
- If the command's not found, does it panic?
Anything that requires *more than the one rootfs input* or anything
about outputs belongs in another part of the spec tests.
If any of these fail, most other parts of the specs will also fail.
*/
func CheckBasicExecution(execEng executor.Executor) {
Convey("SPEC: Attempting launch with a rootfs that doesn't exist should error", func() {
formula := def.Formula{
Inputs: []def.Input{
{
Type: "tar",
Location: "/",
// Funny thing is, the URI isn't even necessarily where the buck stops;
// Remote URIs need not be checked if caches are in play, etc.
// So the hash needs to be set (and needs to be invalid).
URI: "file:///nonexistance/in/its/most/essential/unform.tar.gz",
Hash: "defnot",
},
},
}
Convey("We should get an error from the warehouse", func() {
result := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard).Wait()
So(result.Error, testutil.ShouldBeErrorClass, integrity.WarehouseError)
})
Convey("The job exit code should clearly indicate failure", FailureContinues, func() {
formula.Accents = def.Accents{
Entrypoint: []string{"echo", "echococo"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldNotBeNil)
// Even though one should clearly also check the error status,
// zero here could be very confusing, so jobs that error before start should be -1.
So(job.Wait().ExitCode, ShouldEqual, -1)
})
})
Convey("SPEC: Launching a command with a working rootfs should work", func() {
formula := getBaseFormula()
Convey("The executor should be able to invoke echo", FailureContinues, func() {
formula.Accents = def.Accents{
Entrypoint: []string{"echo", "echococo"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
// note that we can read output concurrently.
// no need to wait for job done.
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldEqual, "echococo\n")
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 0)
})
Convey("The executor should be able to check exit codes", func() {
formula.Accents = def.Accents{
Entrypoint: []string{"sh", "-c", "exit 14"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job, ShouldNotBeNil)
So(job.Wait().Error, ShouldBeNil)
So(job.Wait().ExitCode, ShouldEqual, 14)
})
Convey("The executor should report command not found clearly", FailureContinues, func() {
formula.Accents = def.Accents{
Entrypoint: []string{"not a command"},
}
job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
So(job.Wait().Error, testutil.ShouldBeErrorClass, executor.NoSuchCommandError)
So(job.Wait().ExitCode, ShouldEqual, -1)
msg, err := ioutil.ReadAll(job.OutputReader())
So(err, ShouldBeNil)
So(string(msg), ShouldEqual, "")
})
})
}