func (factory *gardenContainerSpecFactory) BuildTaskContainerSpec(
spec TaskContainerSpec,
gardenSpec garden.ContainerSpec,
cancel <-chan os.Signal,
delegate ImageFetchingDelegate,
id Identifier,
metadata Metadata,
workerClient Client,
) (garden.ContainerSpec, error) {
if spec.ImageResourcePointer == nil {
gardenSpec.RootFSPath = spec.Image
}
gardenSpec.Privileged = spec.Privileged
var err error
gardenSpec, err = factory.createVolumes(gardenSpec, spec.Inputs)
if err != nil {
return gardenSpec, err
}
for _, mount := range spec.Outputs {
volume := mount.Volume
gardenSpec.BindMounts = append(gardenSpec.BindMounts, garden.BindMount{
SrcPath: volume.Path(),
DstPath: mount.MountPath,
Mode: garden.BindMountModeRW,
})
factory.volumeHandles = append(factory.volumeHandles, volume.Handle())
factory.volumeMounts[volume.Handle()] = mount.MountPath
}
return gardenSpec, nil
}
func (s *GardenServer) handleCreate(w http.ResponseWriter, r *http.Request) {
var spec garden.ContainerSpec
if !s.readRequest(&spec, w, r) {
return
}
hLog := s.logger.Session("create", lager.Data{
"request": spec,
})
if spec.GraceTime == 0 {
spec.GraceTime = s.containerGraceTime
}
hLog.Debug("creating")
container, err := s.backend.Create(spec)
if err != nil {
s.writeError(w, err, hLog)
return
}
hLog.Info("created")
s.bomberman.Strap(container)
s.writeResponse(w, &struct{ Handle string }{
Handle: container.Handle(),
})
}
func (p *LinuxResourcePool) Acquire(spec garden.ContainerSpec) (linux_backend.LinuxContainerSpec, error) {
id := <-p.containerIDs
containerPath := path.Join(p.depotPath, id)
pLog := p.logger.Session(id)
pLog.Info("creating")
resources, err := p.acquirePoolResources(spec, id)
if err != nil {
return linux_backend.LinuxContainerSpec{}, err
}
defer cleanup(&err, func() {
p.releasePoolResources(resources)
})
pLog.Info("acquired-pool-resources")
handle := getHandle(spec.Handle, id)
var quota int64 = int64(spec.Limits.Disk.ByteHard)
if quota == 0 {
quota = math.MaxInt64
}
containerRootFSPath, rootFSEnv, err := p.acquireSystemResources(id, handle, containerPath, spec.RootFSPath, resources, spec.BindMounts, quota, pLog)
if err != nil {
return linux_backend.LinuxContainerSpec{}, err
}
pLog.Info("created")
specEnv, err := process.NewEnv(spec.Env)
if err != nil {
p.tryReleaseSystemResources(p.logger, id)
return linux_backend.LinuxContainerSpec{}, err
}
pLog.Debug("calculate-environment", lager.Data{
"rootfs-env": rootFSEnv,
})
spec.Env = rootFSEnv.Merge(specEnv).Array()
spec.Handle = handle
return linux_backend.LinuxContainerSpec{
ID: id,
ContainerPath: containerPath,
ContainerRootFSPath: containerRootFSPath,
Resources: resources,
Events: []string{},
Version: p.currentContainerVersion,
State: linux_backend.StateBorn,
ContainerSpec: spec,
}, nil
}
func (factory *gardenContainerSpecFactory) BuildResourceContainerSpec(
spec ResourceTypeContainerSpec,
gardenSpec garden.ContainerSpec,
resourceTypes []atc.WorkerResourceType,
) (garden.ContainerSpec, error) {
if len(spec.Mounts) > 0 && spec.Cache.Volume != nil {
return gardenSpec, errors.New("a container may not have mounts and a cache")
}
gardenSpec.Privileged = true
gardenSpec.Env = append(gardenSpec.Env, spec.Env...)
if spec.Ephemeral {
gardenSpec.Properties[ephemeralPropertyName] = "true"
}
if spec.Cache.Volume != nil && spec.Cache.MountPath != "" {
gardenSpec.BindMounts = []garden.BindMount{
{
SrcPath: spec.Cache.Volume.Path(),
DstPath: spec.Cache.MountPath,
Mode: garden.BindMountModeRW,
},
}
factory.volumeHandles = append(factory.volumeHandles, spec.Cache.Volume.Handle())
factory.volumeMounts[spec.Cache.Volume.Handle()] = spec.Cache.MountPath
}
var err error
gardenSpec, err = factory.createVolumes(gardenSpec, spec.Mounts)
if err != nil {
return gardenSpec, err
}
if spec.ImageResourcePointer == nil {
for _, t := range resourceTypes {
if t.Type == spec.Type {
gardenSpec.RootFSPath = t.Image
return gardenSpec, nil
}
}
return gardenSpec, ErrUnsupportedResourceType
}
return gardenSpec, nil
}
func (factory *gardenContainerSpecFactory) createVolumes(containerSpec garden.ContainerSpec, mounts []VolumeMount) (garden.ContainerSpec, error) {
for _, mount := range mounts {
cowVolume, err := factory.baggageclaimClient.CreateVolume(factory.logger, baggageclaim.VolumeSpec{
Strategy: baggageclaim.COWStrategy{
Parent: mount.Volume,
},
Privileged: containerSpec.Privileged,
TTL: VolumeTTL,
})
if err != nil {
return containerSpec, err
}
factory.releaseAfterCreate = append(factory.releaseAfterCreate, cowVolume)
containerSpec.BindMounts = append(containerSpec.BindMounts, garden.BindMount{
SrcPath: cowVolume.Path(),
DstPath: mount.MountPath,
Mode: garden.BindMountModeRW,
})
factory.volumeHandles = append(factory.volumeHandles, cowVolume.Handle())
factory.volumeMounts[cowVolume.Handle()] = mount.MountPath
}
return containerSpec, nil
}
func createContainer(containerSpec garden.ContainerSpec) garden.Container {
handle, err := uuid.NewV4()
Expect(err).ShouldNot(HaveOccurred())
containerSpec.Handle = handle.String()
container, err := client.Create(containerSpec)
Expect(err).ShouldNot(HaveOccurred())
err = StreamIn(container)
Expect(err).ShouldNot(HaveOccurred())
return container
}
func (s *GardenServer) handleCreate(w http.ResponseWriter, r *http.Request) {
var spec garden.ContainerSpec
if !s.readRequest(&spec, w, r) {
return
}
hLog := s.logger.Session("create", lager.Data{
"request": containerDebugInfo{
Handle: spec.Handle,
GraceTime: spec.GraceTime,
RootFSPath: spec.RootFSPath,
BindMounts: spec.BindMounts,
Network: spec.Network,
Privileged: spec.Privileged,
Limits: spec.Limits,
},
})
if spec.GraceTime == 0 {
spec.GraceTime = s.containerGraceTime
}
hLog.Debug("creating")
container, err := s.backend.Create(spec)
if err != nil {
s.writeError(w, err, hLog)
return
}
hLog.Info("created")
s.bomberman.Strap(container)
s.writeResponse(w, &struct{ Handle string }{
Handle: container.Handle(),
})
}
func (worker *gardenWorker) CreateContainer(id Identifier, spec ContainerSpec) (Container, error) {
gardenSpec := garden.ContainerSpec{
Properties: id.gardenProperties(),
}
dance:
switch s := spec.(type) {
case ResourceTypeContainerSpec:
gardenSpec.Privileged = true
if s.Ephemeral {
gardenSpec.Properties[ephemeralPropertyName] = "true"
}
for _, t := range worker.resourceTypes {
if t.Type == s.Type {
gardenSpec.RootFSPath = t.Image
break dance
}
}
return nil, ErrUnsupportedResourceType
case TaskContainerSpec:
gardenSpec.RootFSPath = s.Image
gardenSpec.Privileged = s.Privileged
default:
return nil, fmt.Errorf("unknown container spec type: %T (%#v)", s, s)
}
gardenContainer, err := worker.gardenClient.Create(gardenSpec)
if err != nil {
return nil, err
}
return newGardenWorkerContainer(gardenContainer, worker.gardenClient, worker.clock), nil
}
func (g *Gardener) Create(spec garden.ContainerSpec) (garden.Container, error) {
log := g.Logger.Session("create")
if spec.Handle == "" {
spec.Handle = g.UidGenerator.Generate()
}
networkPath, err := g.Networker.Network(log, spec.Handle, spec.Network)
if err != nil {
return nil, err
}
rootFSURL, err := url.Parse(spec.RootFSPath)
if err != nil {
g.Networker.Destroy(g.Logger, spec.Handle)
return nil, err
}
rootFSPath, _, err := g.VolumeCreator.Create(spec.Handle, rootfs_provider.Spec{RootFS: rootFSURL})
if err != nil {
g.Networker.Destroy(g.Logger, spec.Handle)
return nil, err
}
if err := g.Containerizer.Create(log, DesiredContainerSpec{
Handle: spec.Handle,
RootFSPath: rootFSPath,
NetworkPath: networkPath,
}); err != nil {
g.Networker.Destroy(g.Logger, spec.Handle)
return nil, err
}
container, err := g.Lookup(spec.Handle)
if err != nil {
return nil, err
}
for name, value := range spec.Properties {
err := container.SetProperty(name, value)
if err != nil {
return nil, err
}
}
return container, nil
}
func (g *Gardener) Create(spec garden.ContainerSpec) (garden.Container, error) {
if spec.Handle == "" {
spec.Handle = g.UidGenerator.Generate()
}
networkPath, err := g.Networker.Network(spec.Network)
if err != nil {
return nil, err
}
if err := g.Containerizer.Create(DesiredContainerSpec{
Handle: spec.Handle,
NetworkPath: networkPath,
}); err != nil {
return nil, err
}
return g.Lookup(spec.Handle)
}
func (backend *Backend) Create(spec garden.ContainerSpec) (garden.Container, error) {
backend.containersL.Lock()
defer backend.containersL.Unlock()
capacity, err := backend.Capacity()
if err != nil {
return nil, err
}
activeContainers := 0
for _, container := range backend.containers {
if _, ok := container.currentProperties()["concourse:exit-status"]; !ok {
activeContainers++
}
}
if activeContainers >= int(capacity.MaxContainers) {
return nil, atc.WorkerNotCreatedError{errors.New("worker already has the maximum number of active containers")}
}
id := backend.generateContainerID()
if spec.Handle == "" {
spec.Handle = id
}
dir := filepath.Join(backend.containersDir, id)
err = os.MkdirAll(dir, 0755)
if err != nil {
return nil, err
}
container := newContainer(spec, dir)
backend.containers[spec.Handle] = container
return container, nil
}
func (p *LinuxResourcePool) Acquire(spec garden.ContainerSpec) (linux_backend.LinuxContainerSpec, error) {
id := <-p.containerIDs
containerPath := path.Join(p.depotPath, id)
handle := getHandle(spec.Handle, id)
pLog := p.logger.Session("acquire", lager.Data{"handle": handle})
iptablesCh := make(chan error, 1)
go func(iptablesCh chan error) {
pLog.Debug("setup-iptables-starting")
if err := p.filterProvider.ProvideFilter(id).Setup(handle); err != nil {
pLog.Error("setup-iptables-failed", err)
iptablesCh <- fmt.Errorf("resource_pool: set up filter: %v", err)
} else {
pLog.Debug("setup-iptables-ended")
iptablesCh <- nil
}
}(iptablesCh)
pLog.Info("creating")
resources, err := p.acquirePoolResources(spec, id, pLog)
if err != nil {
return linux_backend.LinuxContainerSpec{}, err
}
defer cleanup(&err, func() {
p.releasePoolResources(resources, pLog)
})
pLog.Info("acquired-pool-resources")
pLog.Info("running-graph-cleanup")
if err := p.rootFSProvider.GC(pLog); err != nil {
pLog.Error("graph-cleanup-failed", err)
}
containerRootFSPath, rootFSEnv, err := p.acquireSystemResources(
spec, id, resources, pLog,
)
if err != nil {
return linux_backend.LinuxContainerSpec{}, err
}
err = <-iptablesCh
if err != nil {
p.tryReleaseSystemResources(p.logger, id)
return linux_backend.LinuxContainerSpec{}, err
}
pLog.Info("created")
specEnv, err := process.NewEnv(spec.Env)
if err != nil {
p.tryReleaseSystemResources(p.logger, id)
return linux_backend.LinuxContainerSpec{}, err
}
spec.Env = rootFSEnv.Merge(specEnv).Array()
spec.Handle = handle
return linux_backend.LinuxContainerSpec{
ID: id,
ContainerPath: containerPath,
ContainerRootFSPath: containerRootFSPath,
Resources: resources,
Events: []string{},
Version: p.currentContainerVersion,
State: linux_backend.StateBorn,
ContainerSpec: spec,
}, nil
}
func (c *RuncContainerCreator) Create(spec garden.ContainerSpec) (*Container, error) {
dir, err := c.Depot.Create()
if err != nil {
return nil, fmt.Errorf("create depot dir: %s", err)
}
if len(spec.RootFSPath) == 0 {
spec.RootFSPath = c.DefaultRootfs
}
rootfs, err := url.Parse(spec.RootFSPath)
if err != nil {
return nil, fmt.Errorf("create: not a valid rootfs path: %s", err)
}
if _, err := exec.Command("cp", "-r", rootfs.Path, path.Join(dir, "rootfs")).CombinedOutput(); err != nil {
return nil, fmt.Errorf("create: copy rootfs: %s", err)
}
runcSpec := runc.PortableSpec{
Version: "0.1",
OS: runtime.GOOS,
Arch: runtime.GOARCH,
Cpus: 1.1,
Memory: 1024,
Root: runc.Root{
Path: "rootfs",
Readonly: false,
},
Namespaces: []runc.Namespace{
{
Type: "process",
},
{
Type: "network",
},
{
Type: "mount",
},
{
Type: "ipc",
},
{
Type: "uts",
},
},
Devices: []string{
"null",
"random",
"full",
"tty",
"zero",
"urandom",
},
Mounts: []runc.Mount{
{
Type: "proc",
Source: "proc",
Destination: "/proc",
Options: "",
},
{
Type: "tmpfs",
Source: "tmpfs",
Destination: "/dev",
Options: "nosuid,strictatime,mode=755,size=65536k",
},
{
Type: "devpts",
Source: "devpts",
Destination: "/dev/pts",
Options: "nosuid,noexec,newinstance,ptmxmode=0666,mode=0620,gid=5",
},
{
Type: "tmpfs",
Source: "shm",
Destination: "/dev/shm",
Options: "nosuid,noexec,nodev,mode=1777,size=65536k",
},
{
Type: "mqueue",
Source: "mqueue",
Destination: "/dev/mqueue",
Options: "nosuid,noexec,nodev",
},
{
Type: "sysfs",
Source: "sysfs",
Destination: "/sys",
Options: "nosuid,noexec,nodev",
},
{
Type: "bind",
Source: c.InitdPath,
Destination: "/garden-bin/initd",
Options: "bind",
}, {
Type: "bind",
Source: path.Join(dir, "run"),
Destination: "/run/garden",
Options: "bind",
}},
Processes: []*runc.Process{{
// User: "******",
// Args: []string{
// "/bin/ls", "-lR", "/garden-bin",
// },
// }},
User: "******",
Args: []string{
"/garden-bin/initd",
"-socket", "/run/garden/initd.sock",
"-unmountAfterListening", "/run/garden",
},
}},
}
data, err := json.MarshalIndent(&runcSpec, "", "\t")
if err != nil {
return nil, fmt.Errorf("create: marshal runc spec: %s", err)
}
err = ioutil.WriteFile(path.Join(dir, "container.json"), data, 0700)
if err != nil {
return nil, fmt.Errorf("create: write runc spec: %s", err)
}
os.Setenv("CGO_ENABLED", "1")
runcBin, err := gexec.Build("github.com/opencontainers/runc")
if err != nil {
return nil, fmt.Errorf("create: build runc: %s", err)
}
runcCommand := exec.Command(runcBin)
runcCommand.Dir = dir
if err := c.CommandRunner.Start(runcCommand); err != nil {
return nil, fmt.Errorf("create: start runc container: %s", err)
}
time.Sleep(2 * time.Second)
return &Container{
LimitsHandler: &LimitsHandler{},
StreamHandler: &StreamHandler{},
InfoHandler: &InfoHandler{
Spec: spec,
ContainerPath: dir,
PropsHandler: &PropsHandler{},
},
NetHandler: &NetHandler{
Chain: c.Chain,
PortPool: c.PortPool,
},
RunHandler: &RunHandler{
ProcessTracker: process_tracker.New(dir, c.CommandRunner),
ContainerCmd: &doshcmd{
Path: filepath.Join(dir, "bin", "dosh"),
InitdSock: filepath.Join(dir, "run", "initd.sock"),
},
},
}, nil
}
volumizer.VolumizeReturns("the-volumized-rootfs-path", nil)
containerizer.CreateStub = func(spec gardener.DesiredContainerSpec) error {
return nil
}
gdnr = &gardener.Gardener{
Networker: networker,
Volumizer: volumizer,
Containerizer: containerizer,
}
})
Describe("creating a container", func() {
var (
createdContainer garden.Container
spec garden.ContainerSpec
)
BeforeEach(func() {
spec = garden.ContainerSpec{}
})
JustBeforeEach(func() {
var err error
createdContainer, err = gdnr.Create(spec)
Expect(err).NotTo(HaveOccurred())
})
It("passes the rootfs provided by the volumizer to the containerizer", func() {
Expect(containerizer.CreateArgsForCall(0).RootFSPath).To(Equal("the-volumized-rootfs-path"))
package lifecycle_test
import (
"github.com/cloudfoundry-incubator/garden"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
)
var _ = Describe("Logging", func() {
var container garden.Container
var containerSpec garden.ContainerSpec
BeforeEach(func() {
containerSpec = garden.ContainerSpec{}
})
JustBeforeEach(func() {
var err error
client = startGarden()
container, err = client.Create(containerSpec)
Expect(err).ToNot(HaveOccurred())
})
Context("when container is created", func() {
BeforeEach(func() {
containerSpec = garden.ContainerSpec{
Handle: "kumquat",
Env: []string{"PASSWORD=MY_SECRET"},
Properties: garden.Properties{
func (exchanger exchanger) CreateInGarden(logger lager.Logger, gardenClient GardenClient, executorContainer executor.Container) (executor.Container, error) {
logger = logger.Session("create-in-garden", lager.Data{"container-guid": executorContainer.Guid})
containerSpec := garden.ContainerSpec{
Handle: executorContainer.Guid,
Privileged: executorContainer.Privileged,
RootFSPath: executorContainer.RootFSPath,
}
if executorContainer.MemoryMB != 0 {
logger.Debug("setting-up-memory-limits")
containerSpec.Limits.Memory.LimitInBytes = uint64(executorContainer.MemoryMB * 1024 * 1024)
}
logger.Debug("setting-up-disk-limits")
gardenScope := garden.DiskLimitScopeExclusive
if executorContainer.DiskScope == executor.TotalDiskLimit {
gardenScope = garden.DiskLimitScopeTotal
}
containerSpec.Limits.Disk = garden.DiskLimits{
ByteHard: uint64(executorContainer.DiskMB * 1024 * 1024),
InodeHard: exchanger.containerInodeLimit,
Scope: gardenScope,
}
logger.Debug("setting-up-cpu-limits")
containerSpec.Limits.CPU.LimitInShares = uint64(float64(exchanger.containerMaxCPUShares) * float64(executorContainer.CPUWeight) / 100.0)
logJson, err := json.Marshal(executorContainer.LogConfig)
if err != nil {
logger.Error("failed-marshal-log", err)
return executor.Container{}, err
}
metricsConfigJson, err := json.Marshal(executorContainer.MetricsConfig)
if err != nil {
logger.Error("failed-marshal-metrics-config", err)
return executor.Container{}, err
}
resultJson, err := json.Marshal(executorContainer.RunResult)
if err != nil {
logger.Error("failed-marshal-run-result", err)
return executor.Container{}, err
}
containerSpec.Properties = garden.Properties{
ContainerOwnerProperty: exchanger.containerOwnerName,
ContainerStateProperty: string(executorContainer.State),
ContainerAllocatedAtProperty: fmt.Sprintf("%d", executorContainer.AllocatedAt),
ContainerStartTimeoutProperty: fmt.Sprintf("%d", executorContainer.StartTimeout),
ContainerRootfsProperty: executorContainer.RootFSPath,
ContainerLogProperty: string(logJson),
ContainerMetricsConfigProperty: string(metricsConfigJson),
ContainerResultProperty: string(resultJson),
ContainerMemoryMBProperty: fmt.Sprintf("%d", executorContainer.MemoryMB),
ContainerDiskMBProperty: fmt.Sprintf("%d", executorContainer.DiskMB),
ContainerCPUWeightProperty: fmt.Sprintf("%d", executorContainer.CPUWeight),
}
for name, value := range executorContainer.Tags {
containerSpec.Properties[TagPropertyPrefix+name] = value
}
for _, env := range executorContainer.Env {
containerSpec.Env = append(containerSpec.Env, env.Name+"="+env.Value)
}
for _, securityRule := range executorContainer.EgressRules {
if err := securityRule.Validate(); err != nil {
logger.Error("invalid-security-rule", err, lager.Data{"security_group_rule": securityRule})
return executor.Container{}, executor.ErrInvalidSecurityGroup
}
}
logger.Debug("creating-garden-container")
gardenContainer, err := gardenClient.Create(containerSpec)
if err != nil {
logger.Error("failed-creating-garden-container", err)
return executor.Container{}, err
}
logger.Debug("succeeded-creating-garden-container")
if executorContainer.Ports != nil {
actualPortMappings := make([]executor.PortMapping, len(executorContainer.Ports))
logger.Debug("setting-up-ports")
for i, ports := range executorContainer.Ports {
actualHostPort, actualContainerPort, err := gardenContainer.NetIn(uint32(ports.HostPort), uint32(ports.ContainerPort))
if err != nil {
logger.Error("failed-setting-up-ports", err)
exchanger.destroyContainer(logger, gardenClient, gardenContainer)
return executor.Container{}, err
}
actualPortMappings[i].ContainerPort = uint16(actualContainerPort)
actualPortMappings[i].HostPort = uint16(actualHostPort)
}
logger.Debug("succeeded-setting-up-ports")
executorContainer.Ports = actualPortMappings
}
for _, securityRule := range executorContainer.EgressRules {
netOutRule, err := securityGroupRuleToNetOutRule(securityRule)
if err != nil {
logger.Error("failed-to-build-net-out-rule", err, lager.Data{"security_group_rule": securityRule})
return executor.Container{}, err
}
logger.Debug("setting-up-net-out")
err = gardenContainer.NetOut(netOutRule)
if err != nil {
logger.Error("failed-setting-up-net-out", err, lager.Data{"net-out-rule": netOutRule})
exchanger.destroyContainer(logger, gardenClient, gardenContainer)
return executor.Container{}, err
}
logger.Debug("succeeded-setting-up-net-out")
}
logger.Debug("getting-garden-container-info")
info, err := gardenContainer.Info()
if err != nil {
logger.Error("failed-getting-garden-container-info", err)
gardenErr := gardenClient.Destroy(gardenContainer.Handle())
if gardenErr != nil {
logger.Error("failed-destroy-garden-container", gardenErr)
}
return executor.Container{}, err
}
logger.Debug("succeeded-getting-garden-container-info")
executorContainer.ExternalIP = info.ExternalIP
return executorContainer, nil
}
