Exemple #1
0
func TwerkCommandPattern(stdin io.Reader, stdout, stderr io.Writer) cli.Command {
	return cli.Command{
		Name:  "twerk",
		Usage: "Run one-time-use interactive (thus nonrepeatable!) command.  All the defaults are filled in for you.  Great for experimentation.",
		Action: func(ctx *cli.Context) {
			executor := executordispatch.Get("chroot")
			formula := def.Formula{
				Inputs: []def.Input{{
					Type:      "tar",
					MountPath: "/",
					Hash:      "uJRF46th6rYHt0zt_n3fcDuBfGFVPS6lzRZla5hv6iDoh5DVVzxUTMMzENfPoboL",
					Warehouses: []string{
						"http+ca://repeatr.s3.amazonaws.com/assets/",
					},
				}},
				Action: def.Action{
					Entrypoint: []string{"bash", "-c", "echo hallo ; pwd ; ls -la ; bash"},
				},
			}

			// TODO bonus points if you eventually can get the default mode to have no setuid binaries, in addition to making a spare user and dropping privs immediately.

			fmt.Fprintln(ctx.App.Writer, "launchin")
			job := executor.Start(formula, def.JobID(guid.New()), stdin, ctx.App.Writer)
			go io.Copy(stdout, job.Outputs().Reader(1))
			go io.Copy(stderr, job.Outputs().Reader(2))
			result := job.Wait()
			if result.Error != nil {
				fmt.Fprintf(ctx.App.Writer, "error: %s\n", result.Error)
			} else {
				fmt.Fprintf(ctx.App.Writer, "done; exit code %d\n", result.ExitCode)
			}
		},
	}
}
Exemple #2
0
func soExpectSuccessAndOutput(execEng executor.Executor, formula def.Formula, output string) {
	job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
	So(job, ShouldNotBeNil)
	So(job.Wait().Error, ShouldBeNil)
	So(job.Wait().ExitCode, ShouldEqual, 0)
	msg, err := ioutil.ReadAll(job.OutputReader())
	So(err, ShouldBeNil)
	So(string(msg), ShouldEqual, output)
}
Exemple #3
0
func CheckFilesystemContainment(execEng executor.Executor) {
	Convey("SPEC: Launching with multiple inputs should work", func() {
		formula := getBaseFormula()

		Convey("Launch should succeed", func() {
			filefixture.Beta.Create("./fixture/beta")
			formula.Inputs = append(formula.Inputs, (def.Input{
				Name:       "2-input-test",
				Type:       "dir",
				Hash:       filefixture.Beta_Hash,
				Warehouses: []string{"./fixture/beta"},
				MountPath:  "/data/test",
			}))

			formula.Action = def.Action{
				Entrypoint: []string{"/bin/true"},
			}
			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldBeNil)
			So(job.Wait().ExitCode, ShouldEqual, 0)

			Convey("Commands inside the job should be able to see the mounted files", FailureContinues, func() {
				formula.Action = def.Action{
					Entrypoint: []string{"ls", "/data/test"},
				}

				job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
				So(job, ShouldNotBeNil)
				So(job.Wait().Error, ShouldBeNil)
				So(job.Wait().ExitCode, ShouldEqual, 0)
				msg, err := ioutil.ReadAll(job.OutputReader())
				So(err, ShouldBeNil)
				So(string(msg), ShouldEqual, "1\n2\n3\n")
			})
		})
	})
}
Exemple #4
0
func CheckEnvBehavior(execEng executor.Executor) {
	// NOTE the chroot executor currently uses `def.ValidateAll` which effectively *does not permit you to have an empty $PATH var*.
	// we may decide to change that later -- but there's a correctness versus convenience battle there, and for now, we're going with convenience.

	Convey("SPEC: Env vars should be contained", func() {
		formula := getBaseFormula()
		formula.Action = def.Action{
			Entrypoint: []string{"env"},
		}

		Convey("Env from the parent should not be inherited", func() {
			os.Setenv("REPEATR_TEST_KEY_1", "test value")
			defer os.Unsetenv("REPEATR_TEST_KEY_1") // using unique strings per test anyway, because this is too scary

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldBeNil)
			So(job.Wait().ExitCode, ShouldEqual, 0)
			msg, err := ioutil.ReadAll(job.OutputReader())
			So(err, ShouldBeNil)
			So(strings.Contains("REPEATR_TEST_KEY_1", string(msg)), ShouldBeFalse)
		})

		Convey("Env specified with the job should be applied", func() {
			formula.Action.Env = make(map[string]string)
			formula.Action.Env["REPEATR_TEST_KEY_2"] = "test value"

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldBeNil)
			So(job.Wait().ExitCode, ShouldEqual, 0)
			msg, err := ioutil.ReadAll(job.OutputReader())
			So(err, ShouldBeNil)
			So(string(msg), ShouldContainSubstring, "REPEATR_TEST_KEY_2=test value")
		})

	})
}
Exemple #5
0
func CheckPwdBehavior(execEng executor.Executor) {
	Convey("SPEC: Working directory should be contained", func() {
		formula := getBaseFormula()

		Convey("The default pwd should be the root", func() {
			// This test exists because of a peculularly terrifying fact about chroots:
			// If you spawn a new process with the chroot without setting its current working dir,
			// the cwd is still *whatever it inherits*.  And even if the process can't reach there
			// starting from "/", it can still *walk deeper from that cwd*.
			formula.Action = def.Action{
				//Entrypoint: []string{"find", "-maxdepth", "5"}, // if you goofed, during test runs this will show you the executor's workspace!
				//Entrypoint: []string{"bash", "-c", "find -maxdepth 5 ; echo --- ; echo \"$PWD\" ; cd \"$(echo \"$PWD\" | sed 's/^(unreachable)//')\" ; ls"}, // this demo's that you can't actually cd back to it, though.
				Entrypoint: []string{"pwd"},
			}

			soExpectSuccessAndOutput(execEng, formula,
				"/\n",
			)
		})

		Convey("Setting another cwd should work", func() {
			formula.Action = def.Action{
				Cwd:        "/usr",
				Entrypoint: []string{"pwd"},
			}

			soExpectSuccessAndOutput(execEng, formula,
				"/usr\n",
			)
		})

		Convey("Setting a nonexistent cwd should fail to launch", FailureContinues, func() {
			formula.Action = def.Action{
				Cwd:        "/does/not/exist/by/any/means",
				Entrypoint: []string{"pwd"},
			}

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldNotBeNil)
			So(job.Wait().Error, testutil.ShouldBeErrorClass, executor.TaskExecError)
			So(job.Wait().ExitCode, ShouldEqual, -1)
			msg, err := ioutil.ReadAll(job.OutputReader())
			So(err, ShouldBeNil)
			So(string(msg), ShouldEqual, "")
		})
	})
}
Exemple #6
0
func makeWriteController(warehouseCoords integrity.SiloURI) StreamingWarehouseWriteController {
	u, err := url.Parse(string(warehouseCoords))
	if err != nil {
		panic(integrity.ConfigError.New("failed to parse URI: %s", err))
	}
	controller := &fileWarehouseWriteController{
		pathPrefix: u.Path,
	}
	switch u.Scheme {
	case "file+ca":
		controller.ctntAddr = true
		fallthrough
	case "file":
		// Pick a random upload path
		controller.pathPrefix = filepath.Join(u.Host, controller.pathPrefix) // file uris don't have hosts
		if controller.ctntAddr {
			controller.tmpPath = filepath.Join(controller.pathPrefix, ".tmp.upload."+guid.New())
		} else {
			controller.tmpPath = filepath.Join(path.Dir(controller.pathPrefix), ".tmp.upload."+path.Base(controller.pathPrefix)+"."+guid.New())
		}
		// Check if warehouse path exists.
		// Warehouse is expected to exist already; transmats
		//  should *not* create one whimsically, that's someone else's responsibility.
		warehouseBasePath := filepath.Dir(controller.tmpPath)
		if _, err := os.Stat(warehouseBasePath); err != nil {
			panic(integrity.WarehouseConnectionError.New("Warehouse unavailable: %q %s", warehouseBasePath, err))
		}
		// Open file to shovel data into
		file, err := os.OpenFile(controller.tmpPath, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			panic(integrity.WarehouseConnectionError.New("Unable to write %q: %s", controller.tmpPath, err))
		}
		controller.stream = file
		return controller
	case "http+ca":
		fallthrough
	case "http":
		fallthrough
	case "https+ca":
		fallthrough
	case "https":
		panic(integrity.ConfigError.New("http transports are only supported for read-only use"))
	case "":
		panic(integrity.ConfigError.New("missing scheme in warehouse URI; need a prefix, e.g. \"file://\" or \"http://\""))
	default:
		panic(integrity.ConfigError.New("unsupported scheme in warehouse URI: %q", u.Scheme))
	}
}
Exemple #7
0
func (s *Scheduler) Schedule(f def.Formula) (def.JobID, <-chan def.Job) {
	id := def.JobID(guid.New())

	h := &hold{
		id:       id,
		forumla:  f,
		response: make(chan def.Job),
	}

	// Non-blocking send, will panic if scheduler queue is full
	select {
	case s.queue <- h:
	default:
		panic(scheduler.QueueFullError)
	}

	return id, h.response
}
func TestCoreCompliance(t *testing.T) {
	if _, err := s3gof3r.EnvKeys(); err != nil {
		t.Skipf("skipping s3 output tests; no s3 credentials loaded (err: %s)", err)
	}

	// group all effects of this test run under one "dir" for human reader sanity and cleanup in extremis.
	testRunGuid := guid.New()

	Convey("Spec Compliance: S3 Transmat", t, testutil.WithTmpdir(func() {
		// scanning
		tests.CheckScanWithoutMutation(Kind, New)
		tests.CheckScanProducesConsistentHash(Kind, New)
		tests.CheckScanProducesDistinctHashes(Kind, New)
		tests.CheckScanEmptyIsCalm(Kind, New)
		tests.CheckScanWithFilters(Kind, New)
		// round-trip
		tests.CheckRoundTrip(Kind, New, "s3://repeatr-test/test-"+testRunGuid+"/rt/obj.tar", "literal path")
		tests.CheckRoundTrip(Kind, New, "s3+splay://repeatr-test/test-"+testRunGuid+"/rt-splay/heap/", "content addressible path")
	}))
}
Exemple #9
0
func (t S3Transmat) Scan(
	kind integrity.TransmatKind,
	subjectPath string,
	siloURIs []integrity.SiloURI,
	options ...integrity.MaterializerConfigurer,
) integrity.CommitID {
	var commitID integrity.CommitID
	try.Do(func() {
		// Basic validation and config
		config := integrity.EvaluateConfig(options...)
		if kind != Kind {
			panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
		}

		// If scan area doesn't exist, bail immediately.
		// No need to even start dialing warehouses if we've got nothing for em.
		_, err := os.Stat(subjectPath)
		if err != nil {
			if os.IsNotExist(err) {
				return // empty commitID
			} else {
				panic(err)
			}
		}

		// load keys from env
		// TODO someday URIs should grow smart enough to control this in a more general fashion -- but for now, host ENV is actually pretty feasible and plays easily with others.
		keys, err := s3gof3r.EnvKeys()
		if err != nil {
			panic(S3CredentialsMissingError.Wrap(err))
		}

		// Parse URI; Find warehouses; Open output streams for writing.
		// Since these are all behaving as just one `io.Writer` stream, this could maybe be factored out.
		// Error handling is currently "anything -> panic".  This should probably be more resilient.  (That might need another refactor so we have an upload call per remote.)
		// TODO : both this and the tar code that has a similar single stream idea should use an interface
		//  And that interface should have a concept of mv so we can make atomic commits.
		//  I'm not doing multiple URIs here until we get that, because the io.Writer interface just
		//   doesn't cut it like it did for tars (and really, it's ignoring a major issue to use it there, too).
		//  ...F**k it, we're gonna do it
		controllers := make([]*s3warehousePut, 0)
		writers := make([]io.Writer, 0) // this is dumb, but we end up making one of these to satisfy the type conversation for MultiWriter anyway
		for _, givenURI := range siloURIs {
			u, err := url.Parse(string(givenURI))
			if err != nil {
				panic(integrity.ConfigError.New("failed to parse URI: %s", err))
			}
			controller := &s3warehousePut{}
			controller.bucketName = u.Host
			controller.pathPrefix = u.Path
			var ctntAddr bool
			switch u.Scheme {
			case "s3":
				ctntAddr = false
			case "s3+splay":
				ctntAddr = true
			default:
				panic(integrity.ConfigError.New("unrecognized scheme: %q", u.Scheme))
			}
			// dial it and initialize writer to s3!
			// if the URI indicated splay behavior, first stream data to {$bucketName}:{dirname($storePath)}/.tmp.upload.{basename($storePath)}.{random()};
			// this allows us to start uploading before the final hash is determined and relocate it later.
			// for direct paths, upload into place, because aws already manages atomicity at that scale (and they don't have a rename or copy operation that's free, because uh...?  no time to implement it since 2006, apparently).
			controller.keys = keys
			if ctntAddr {
				controller.tmpPath = path.Join(
					path.Dir(controller.pathPrefix),
					".tmp.upload."+path.Base(controller.pathPrefix)+"."+guid.New(),
				)
				controller.stream = makeS3writer(controller.bucketName, controller.tmpPath, keys)
			} else {
				controller.stream = makeS3writer(controller.bucketName, controller.pathPrefix, keys)
			}
			controllers = append(controllers, controller)
			writers = append(writers, controller.stream)
		}
		stream := io.MultiWriter(writers...)
		if len(writers) < 1 {
			stream = ioutil.Discard
		}

		// walk, fwrite, hash
		commitID = integrity.CommitID(tartrans.Save(stream, subjectPath, config.FilterSet, hasherFactory))

		// commit
		for _, controller := range controllers {
			controller.Commit(string(commitID))
		}
	}).Catch(integrity.Error, func(err *errors.Error) {
		panic(err)
	}).CatchAll(func(err error) {
		panic(integrity.UnknownError.Wrap(err))
	}).Done()
	return commitID
}
/*
	Check the basics of exec:
	  - Does it work at all?
	  - Can we see error codes?
	  - Can we stream stdout?
	  - If there's no rootfs, does it panic?
	  - If the command's not found, does it panic?

	Anything that requires *more than the one rootfs input* or anything
	about outputs belongs in another part of the spec tests.

	If any of these fail, most other parts of the specs will also fail.
*/
func CheckBasicExecution(execEng executor.Executor) {
	Convey("SPEC: Attempting launch with a rootfs that doesn't exist should error", func() {
		formula := def.Formula{
			Inputs: []def.Input{
				{
					Type:     "tar",
					Location: "/",
					// Funny thing is, the URI isn't even necessarily where the buck stops;
					// Remote URIs need not be checked if caches are in play, etc.
					// So the hash needs to be set (and needs to be invalid).
					URI:  "file:///nonexistance/in/its/most/essential/unform.tar.gz",
					Hash: "defnot",
				},
			},
		}

		Convey("We should get an error from the warehouse", func() {
			result := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard).Wait()
			So(result.Error, testutil.ShouldBeErrorClass, integrity.WarehouseError)
		})

		Convey("The job exit code should clearly indicate failure", FailureContinues, func() {
			formula.Accents = def.Accents{
				Entrypoint: []string{"echo", "echococo"},
			}
			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldNotBeNil)
			// Even though one should clearly also check the error status,
			//  zero here could be very confusing, so jobs that error before start should be -1.
			So(job.Wait().ExitCode, ShouldEqual, -1)
		})
	})

	Convey("SPEC: Launching a command with a working rootfs should work", func() {
		formula := getBaseFormula()

		Convey("The executor should be able to invoke echo", FailureContinues, func() {
			formula.Accents = def.Accents{
				Entrypoint: []string{"echo", "echococo"},
			}

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			// note that we can read output concurrently.
			// no need to wait for job done.
			msg, err := ioutil.ReadAll(job.OutputReader())
			So(err, ShouldBeNil)
			So(string(msg), ShouldEqual, "echococo\n")
			So(job.Wait().Error, ShouldBeNil)
			So(job.Wait().ExitCode, ShouldEqual, 0)
		})

		Convey("The executor should be able to check exit codes", func() {
			formula.Accents = def.Accents{
				Entrypoint: []string{"sh", "-c", "exit 14"},
			}

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job, ShouldNotBeNil)
			So(job.Wait().Error, ShouldBeNil)
			So(job.Wait().ExitCode, ShouldEqual, 14)
		})

		Convey("The executor should report command not found clearly", FailureContinues, func() {
			formula.Accents = def.Accents{
				Entrypoint: []string{"not a command"},
			}

			job := execEng.Start(formula, def.JobID(guid.New()), nil, ioutil.Discard)
			So(job.Wait().Error, testutil.ShouldBeErrorClass, executor.NoSuchCommandError)
			So(job.Wait().ExitCode, ShouldEqual, -1)
			msg, err := ioutil.ReadAll(job.OutputReader())
			So(err, ShouldBeNil)
			So(string(msg), ShouldEqual, "")
		})
	})
}