func New(network, addr string, bin, binPath, rootFSPath, graphRoot string, argv ...string) *Runner {
tmpDir := filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
)
if graphRoot == "" {
graphRoot = filepath.Join(tmpDir, "graph")
}
graphPath := filepath.Join(graphRoot, fmt.Sprintf("node-%d", ginkgo.GinkgoParallelNode()))
return &Runner{
network: network,
addr: addr,
bin: bin,
argv: argv,
binPath: binPath,
rootFSPath: rootFSPath,
graphRoot: graphRoot,
graphPath: graphPath,
tmpdir: tmpDir,
}
}
func start(creator RunnerCreator, network, addr string, argv ...string) *RunningGarden {
tmpDir := filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
)
if GraphRoot == "" {
GraphRoot = filepath.Join(tmpDir, "graph")
}
graphPath := filepath.Join(GraphRoot, fmt.Sprintf("node-%d", ginkgo.GinkgoParallelNode()))
r := &RunningGarden{
GraphRoot: GraphRoot,
GraphPath: graphPath,
tmpdir: tmpDir,
logger: lagertest.NewTestLogger("garden-runner"),
Client: client.New(connection.New(network, addr)),
}
c := cmd(tmpDir, graphPath, network, addr, GardenBin, BinPath, RootFSPath, argv...)
r.process = ifrit.Invoke(creator.Create(c))
r.Pid = c.Process.Pid
return r
}
func start(network, addr string, argv ...string) *RunningGarden {
tmpDir := filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
)
Expect(os.MkdirAll(tmpDir, 0755)).To(Succeed())
if GraphRoot == "" {
GraphRoot = filepath.Join(tmpDir, "graph")
}
graphPath := filepath.Join(GraphRoot, fmt.Sprintf("node-%d", ginkgo.GinkgoParallelNode()))
stateDirPath := filepath.Join(tmpDir, "state")
depotPath := filepath.Join(tmpDir, "containers")
snapshotsPath := filepath.Join(tmpDir, "snapshots")
if err := os.MkdirAll(stateDirPath, 0755); err != nil {
Expect(err).ToNot(HaveOccurred())
}
if err := os.MkdirAll(depotPath, 0755); err != nil {
Expect(err).ToNot(HaveOccurred())
}
if err := os.MkdirAll(snapshotsPath, 0755); err != nil {
Expect(err).ToNot(HaveOccurred())
}
MustMountTmpfs(graphPath)
r := &RunningGarden{
GraphRoot: GraphRoot,
GraphPath: graphPath,
StateDirPath: stateDirPath,
DepotPath: depotPath,
SnapshotsPath: snapshotsPath,
tmpdir: tmpDir,
logger: lagertest.NewTestLogger("garden-runner"),
Client: client.New(connection.New(network, addr)),
}
c := cmd(stateDirPath, depotPath, snapshotsPath, graphPath, network, addr, GardenBin, BinPath, RootFSPath, argv...)
r.runner = ginkgomon.New(ginkgomon.Config{
Name: "garden-linux",
Command: c,
AnsiColorCode: "31m",
StartCheck: "garden-linux.started",
StartCheckTimeout: 30 * time.Second,
})
r.process = ifrit.Invoke(r.runner)
r.Pid = c.Process.Pid
return r
}
func cmd(stateDirPath, depotPath, snapshotsPath, graphPath, network, addr, bin, binPath, RootFSPath string, argv ...string) *exec.Cmd {
appendDefaultFlag := func(ar []string, key, value string) []string {
for _, a := range argv {
if a == key {
return ar
}
}
if value != "" {
return append(ar, key, value)
} else {
return append(ar, key)
}
}
hasFlag := func(ar []string, key string) bool {
for _, a := range ar {
if a == key {
return true
}
}
return false
}
gardenArgs := make([]string, len(argv))
copy(gardenArgs, argv)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenNetwork", network)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenAddr", addr)
gardenArgs = appendDefaultFlag(gardenArgs, "--bin", binPath)
if RootFSPath != "" { //rootfs is an optional parameter
gardenArgs = appendDefaultFlag(gardenArgs, "--rootfs", RootFSPath)
}
gardenArgs = appendDefaultFlag(gardenArgs, "--stateDir", stateDirPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--depot", depotPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--snapshots", snapshotsPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--graph", graphPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--logLevel", "debug")
gardenArgs = appendDefaultFlag(gardenArgs, "--networkPool", fmt.Sprintf("10.250.%d.0/24", ginkgo.GinkgoParallelNode()))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolStart", strconv.Itoa(51000+(1000*ginkgo.GinkgoParallelNode())))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolSize", "1000")
gardenArgs = appendDefaultFlag(gardenArgs, "--tag", strconv.Itoa(ginkgo.GinkgoParallelNode()))
if !hasFlag(gardenArgs, "-enableGraphCleanup=false") {
gardenArgs = appendDefaultFlag(gardenArgs, "--enableGraphCleanup", "")
}
gardenArgs = appendDefaultFlag(gardenArgs, "--debugAddr", fmt.Sprintf(":808%d", ginkgo.GinkgoParallelNode()))
return exec.Command(bin, gardenArgs...)
}
func (runner *CLIRunner) generateConfig(storeURLs []string, ccBaseURL string, natsPort int, metricsServerPort int) *config.Config {
tmpFile, err := ioutil.TempFile("/tmp", "hm9000_clirunner")
defer tmpFile.Close()
Ω(err).ShouldNot(HaveOccurred())
runner.configPath = tmpFile.Name()
conf, err := config.DefaultConfig()
Ω(err).ShouldNot(HaveOccurred())
conf.StoreURLs = storeURLs
conf.CCBaseURL = ccBaseURL
conf.NATS[0].Port = natsPort
conf.SenderMessageLimit = 8
conf.MaximumBackoffDelayInHeartbeats = 6
conf.MetricsServerPort = metricsServerPort
conf.MetricsServerUser = "******"
conf.MetricsServerPassword = "******"
conf.StoreMaxConcurrentRequests = 10
conf.ListenerHeartbeatSyncIntervalInMilliseconds = 100
conf.APIServerPort = int(5155 + ginkgo.GinkgoParallelNode())
err = json.NewEncoder(tmpFile).Encode(conf)
Ω(err).ShouldNot(HaveOccurred())
return conf
}
func (maker ComponentMaker) Etcd(argv ...string) ifrit.Runner {
nodeName := fmt.Sprintf("etcd_%d", ginkgo.GinkgoParallelNode())
dataDir := path.Join(os.TempDir(), nodeName)
return ginkgomon.New(ginkgomon.Config{
Name: "etcd",
AnsiColorCode: "31m",
StartCheck: "etcdserver: published",
StartCheckTimeout: 10 * time.Second,
Command: exec.Command(
"etcd",
append([]string{
"--name", nodeName,
"--data-dir", dataDir,
"--listen-client-urls", "https://" + maker.Addresses.Etcd,
"--listen-peer-urls", "http://" + maker.Addresses.EtcdPeer,
"--initial-cluster", nodeName + "=" + "http://" + maker.Addresses.EtcdPeer,
"--initial-advertise-peer-urls", "http://" + maker.Addresses.EtcdPeer,
"--initial-cluster-state", "new",
"--advertise-client-urls", "https://" + maker.Addresses.Etcd,
"--cert-file", maker.SSL.ServerCert,
"--key-file", maker.SSL.ServerKey,
"--ca-file", maker.SSL.CACert,
}, argv...)...,
),
Cleanup: func() {
err := os.RemoveAll(dataDir)
Expect(err).NotTo(HaveOccurred())
},
})
}
func (maker ComponentMaker) RepN(n int, argv ...string) *ginkgomon.Runner {
host, portString, err := net.SplitHostPort(maker.Addresses.Rep)
Expect(err).NotTo(HaveOccurred())
port, err := strconv.Atoi(portString)
Expect(err).NotTo(HaveOccurred())
name := "rep-" + strconv.Itoa(n)
tmpDir, err := ioutil.TempDir(os.TempDir(), "executor")
Expect(err).NotTo(HaveOccurred())
cachePath := path.Join(tmpDir, "cache")
args := append(
[]string{
"-sessionName", name,
"-rootFSProvider", "docker",
"-etcdCluster", "https://" + maker.Addresses.Etcd,
"-bbsAddress", fmt.Sprintf("http://%s", maker.Addresses.BBS),
"-listenAddr", fmt.Sprintf("%s:%d", host, offsetPort(port, n)),
"-cellID", "the-cell-id-" + strconv.Itoa(ginkgo.GinkgoParallelNode()) + "-" + strconv.Itoa(n),
"-pollingInterval", "1s",
"-evacuationPollingInterval", "1s",
"-evacuationTimeout", "1s",
"-lockTTL", "10s",
"-lockRetryInterval", "1s",
"-consulCluster", maker.ConsulCluster(),
"-receptorTaskHandlerURL", "http://" + maker.Addresses.ReceptorTaskHandler,
"-gardenNetwork", "tcp",
"-gardenAddr", maker.Addresses.GardenLinux,
"-containerMaxCpuShares", "1024",
"-cachePath", cachePath,
"-tempDir", tmpDir,
"-logLevel", "debug",
"-etcdCertFile", maker.SSL.ClientCert,
"-etcdKeyFile", maker.SSL.ClientKey,
"-etcdCaFile", maker.SSL.CACert,
},
argv...,
)
for stack, path := range maker.PreloadedStackPathMap {
args = append(args, "-preloadedRootFS", fmt.Sprintf("%s:%s", stack, path))
}
return ginkgomon.New(ginkgomon.Config{
Name: name,
AnsiColorCode: "92m",
StartCheck: `"` + name + `.started"`,
// rep is not started until it can ping an executor; executor can take a
// bit to start, so account for it
StartCheckTimeout: 30 * time.Second,
Command: exec.Command(maker.Artifacts.Executables["rep"], args...),
Cleanup: func() {
os.RemoveAll(tmpDir)
},
})
}
func Start(bin, iodaemonBin, nstarBin string, argv ...string) *RunningGarden {
network := "unix"
addr := fmt.Sprintf("/tmp/garden_%d.sock", GinkgoParallelNode())
tmpDir := filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
)
if GraphRoot == "" {
GraphRoot = filepath.Join(tmpDir, "graph")
}
graphPath := filepath.Join(GraphRoot, fmt.Sprintf("node-%d", ginkgo.GinkgoParallelNode()))
depotDir := filepath.Join(tmpDir, "containers")
MustMountTmpfs(graphPath)
r := &RunningGarden{
DepotDir: depotDir,
GraphRoot: GraphRoot,
GraphPath: graphPath,
tmpdir: tmpDir,
logger: lagertest.NewTestLogger("garden-runner"),
Client: client.New(connection.New(network, addr)),
}
c := cmd(tmpDir, depotDir, graphPath, network, addr, bin, iodaemonBin, nstarBin, TarPath, RootFSPath, argv...)
r.process = ifrit.Invoke(&ginkgomon.Runner{
Name: "guardian",
Command: c,
AnsiColorCode: "31m",
StartCheck: "guardian.started",
StartCheckTimeout: 30 * time.Second,
})
r.Pid = c.Process.Pid
return r
}
func New(network, addr, bin string, argv ...string) *Runner {
return &Runner{
network: network,
addr: addr,
bin: bin,
argv: argv,
tmpdir: filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
),
}
}
func NewGardenRunner(bin, initBin, nstarBin, dadooBin, grootfsBin, rootfs, tarBin, network, address string, argv ...string) GardenRunner {
r := GardenRunner{}
r.Network = network
r.Addr = address
r.TmpDir = filepath.Join(
os.TempDir(),
fmt.Sprintf("test-garden-%d", ginkgo.GinkgoParallelNode()),
)
r.GraphPath = filepath.Join(DataDir, fmt.Sprintf("node-%d", ginkgo.GinkgoParallelNode()))
r.DepotDir = filepath.Join(r.TmpDir, "containers")
MustMountTmpfs(r.GraphPath)
r.Cmd = cmd(r.TmpDir, r.DepotDir, r.GraphPath, r.Network, r.Addr, bin, initBin, nstarBin, dadooBin, grootfsBin, tarBin, rootfs, argv...)
r.Cmd.Env = append(os.Environ(), fmt.Sprintf("TMPDIR=%s", r.TmpDir))
for i, arg := range r.Cmd.Args {
if arg == "--debug-bind-ip" {
r.DebugIp = r.Cmd.Args[i+1]
}
if arg == "--debug-bind-port" {
r.DebugPort, _ = strconv.Atoi(r.Cmd.Args[i+1])
}
}
r.Runner = ginkgomon.New(ginkgomon.Config{
Name: "guardian",
Command: r.Cmd,
AnsiColorCode: "31m",
StartCheck: "guardian.started",
StartCheckTimeout: 30 * time.Second,
})
return r
}
func cmd(tmpdir, depotDir, graphPath, network, addr, bin, iodaemonBin, nstarBin, tarBin, rootFSPath string, argv ...string) *exec.Cmd {
Expect(os.MkdirAll(tmpdir, 0755)).To(Succeed())
snapshotsPath := filepath.Join(tmpdir, "snapshots")
Expect(os.MkdirAll(depotDir, 0755)).To(Succeed())
Expect(os.MkdirAll(snapshotsPath, 0755)).To(Succeed())
appendDefaultFlag := func(ar []string, key, value string) []string {
for _, a := range argv {
if a == key {
return ar
}
}
if value != "" {
return append(ar, key, value)
} else {
return append(ar, key)
}
}
gardenArgs := make([]string, len(argv))
copy(gardenArgs, argv)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenNetwork", network)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenAddr", addr)
gardenArgs = appendDefaultFlag(gardenArgs, "--depot", depotDir)
gardenArgs = appendDefaultFlag(gardenArgs, "--graph", graphPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--tag", fmt.Sprintf("%d", GinkgoParallelNode()))
gardenArgs = appendDefaultFlag(gardenArgs, "--iodaemonBin", iodaemonBin)
gardenArgs = appendDefaultFlag(gardenArgs, "--nstarBin", nstarBin)
gardenArgs = appendDefaultFlag(gardenArgs, "--tarBin", tarBin)
gardenArgs = appendDefaultFlag(gardenArgs, "--logLevel", "debug")
gardenArgs = appendDefaultFlag(gardenArgs, "--debugAddr", fmt.Sprintf(":808%d", ginkgo.GinkgoParallelNode()))
gardenArgs = appendDefaultFlag(gardenArgs, "--rootfs", rootFSPath)
return exec.Command(bin, gardenArgs...)
}
func (r *Runner) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
logger := lagertest.NewTestLogger("garden-runner")
if err := os.MkdirAll(r.tmpdir, 0755); err != nil {
return err
}
depotPath := filepath.Join(r.tmpdir, "containers")
snapshotsPath := filepath.Join(r.tmpdir, "snapshots")
if err := os.MkdirAll(depotPath, 0755); err != nil {
return err
}
if err := os.MkdirAll(snapshotsPath, 0755); err != nil {
return err
}
var appendDefaultFlag = func(ar []string, key, value string) []string {
for _, a := range r.argv {
if a == key {
return ar
}
}
if value != "" {
return append(ar, key, value)
} else {
return append(ar, key)
}
}
var hasFlag = func(ar []string, key string) bool {
for _, a := range ar {
if a == key {
return true
}
}
return false
}
gardenArgs := make([]string, len(r.argv))
copy(gardenArgs, r.argv)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenNetwork", r.network)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenAddr", r.addr)
gardenArgs = appendDefaultFlag(gardenArgs, "--bin", r.binPath)
if r.rootFSPath != "" { //rootfs is an optional parameter
gardenArgs = appendDefaultFlag(gardenArgs, "--rootfs", r.rootFSPath)
}
gardenArgs = appendDefaultFlag(gardenArgs, "--depot", depotPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--snapshots", snapshotsPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--graph", r.graphPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--logLevel", "debug")
gardenArgs = appendDefaultFlag(gardenArgs, "--networkPool", fmt.Sprintf("10.250.%d.0/24", ginkgo.GinkgoParallelNode()))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolStart", strconv.Itoa(51000+(1000*ginkgo.GinkgoParallelNode())))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolSize", "1000")
gardenArgs = appendDefaultFlag(gardenArgs, "--tag", strconv.Itoa(ginkgo.GinkgoParallelNode()))
btrfsIsSupported := strings.EqualFold(os.Getenv("BTRFS_SUPPORTED"), "true")
hasDisabledFlag := hasFlag(gardenArgs, "-disableQuotas=true")
if !btrfsIsSupported && !hasDisabledFlag {
// We should disabled quotas if BTRFS is not supported
gardenArgs = appendDefaultFlag(gardenArgs, "--disableQuotas", "")
}
gardenArgs = appendDefaultFlag(gardenArgs, "--debugAddr", fmt.Sprintf(":808%d", ginkgo.GinkgoParallelNode()))
var signal os.Signal
r.Command = exec.Command(r.bin, gardenArgs...)
process := ifrit.Invoke(&ginkgomon.Runner{
Name: "garden-linux",
Command: r.Command,
AnsiColorCode: "31m",
StartCheck: "garden-linux.started",
StartCheckTimeout: 30 * time.Second,
Cleanup: func() {
if signal == syscall.SIGQUIT {
logger.Info("cleanup-subvolumes")
// remove contents of subvolumes before deleting the subvolume
if err := os.RemoveAll(r.graphPath); err != nil {
logger.Error("remove graph", err)
}
if btrfsIsSupported {
// need to remove subvolumes before cleaning graphpath
subvolumesOutput, err := exec.Command("btrfs", "subvolume", "list", r.graphRoot).CombinedOutput()
logger.Debug(fmt.Sprintf("listing-subvolumes: %s", string(subvolumesOutput)))
if err != nil {
logger.Fatal("listing-subvolumes-error", err)
}
for _, line := range strings.Split(string(subvolumesOutput), "\n") {
fields := strings.Fields(line)
if len(fields) < 1 {
continue
}
subvolumeRelativePath := fields[len(fields)-1]
subvolumeAbsolutePath := filepath.Join(r.graphRoot, subvolumeRelativePath)
if strings.Contains(subvolumeAbsolutePath, r.graphPath) {
if b, err := exec.Command("btrfs", "subvolume", "delete", subvolumeAbsolutePath).CombinedOutput(); err != nil {
logger.Fatal(fmt.Sprintf("deleting-subvolume: %s", string(b)), err)
}
}
}
if err := os.RemoveAll(r.graphPath); err != nil {
logger.Error("remove graph again", err)
}
}
logger.Info("cleanup-tempdirs")
if err := os.RemoveAll(r.tmpdir); err != nil {
logger.Error("cleanup-tempdirs-failed", err, lager.Data{"tmpdir": r.tmpdir})
} else {
logger.Info("tempdirs-removed")
}
}
},
})
close(ready)
for {
select {
case signal = <-signals:
// SIGQUIT means clean up the containers, the garden process (SIGTERM) and the temporary directories
// SIGKILL, SIGTERM and SIGINT are passed through to the garden process
if signal == syscall.SIGQUIT {
logger.Info("received-signal SIGQUIT")
if err := r.destroyContainers(); err != nil {
logger.Error("destroy-containers-failed", err)
return err
}
logger.Info("destroyed-containers")
process.Signal(syscall.SIGTERM)
} else {
logger.Info("received-signal", lager.Data{"signal": signal})
process.Signal(signal)
}
case waitErr := <-process.Wait():
logger.Info("process-exited")
return waitErr
}
}
}
func cmd(tmpdir, graphPath, network, addr, bin, binPath, RootFSPath string, argv ...string) *exec.Cmd {
Expect(os.MkdirAll(tmpdir, 0755)).To(Succeed())
depotPath := filepath.Join(tmpdir, "containers")
snapshotsPath := filepath.Join(tmpdir, "snapshots")
Expect(os.MkdirAll(depotPath, 0755)).To(Succeed())
Expect(os.MkdirAll(snapshotsPath, 0755)).To(Succeed())
appendDefaultFlag := func(ar []string, key, value string) []string {
for _, a := range argv {
if a == key {
return ar
}
}
if value != "" {
return append(ar, key, value)
} else {
return append(ar, key)
}
}
hasFlag := func(ar []string, key string) bool {
for _, a := range ar {
if a == key {
return true
}
}
return false
}
gardenArgs := make([]string, len(argv))
copy(gardenArgs, argv)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenNetwork", network)
gardenArgs = appendDefaultFlag(gardenArgs, "--listenAddr", addr)
gardenArgs = appendDefaultFlag(gardenArgs, "--bin", binPath)
if RootFSPath != "" { //rootfs is an optional parameter
gardenArgs = appendDefaultFlag(gardenArgs, "--rootfs", RootFSPath)
}
gardenArgs = appendDefaultFlag(gardenArgs, "--depot", depotPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--snapshots", snapshotsPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--graph", graphPath)
gardenArgs = appendDefaultFlag(gardenArgs, "--logLevel", "debug")
gardenArgs = appendDefaultFlag(gardenArgs, "--networkPool", fmt.Sprintf("10.250.%d.0/24", ginkgo.GinkgoParallelNode()))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolStart", strconv.Itoa(51000+(1000*ginkgo.GinkgoParallelNode())))
gardenArgs = appendDefaultFlag(gardenArgs, "--portPoolSize", "1000")
gardenArgs = appendDefaultFlag(gardenArgs, "--tag", strconv.Itoa(ginkgo.GinkgoParallelNode()))
btrfsIsSupported := strings.EqualFold(os.Getenv("BTRFS_SUPPORTED"), "true")
hasDisabledFlag := hasFlag(gardenArgs, "-disableQuotas=true")
if !btrfsIsSupported && !hasDisabledFlag {
// We should disabled quotas if BTRFS is not supported
gardenArgs = appendDefaultFlag(gardenArgs, "--disableQuotas", "")
}
gardenArgs = appendDefaultFlag(gardenArgs, "--debugAddr", fmt.Sprintf(":808%d", ginkgo.GinkgoParallelNode()))
return exec.Command(bin, gardenArgs...)
}