func NewValidDesiredLRP(guid string) *models.DesiredLRP {
myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
modTag := models.NewModificationTag("epoch", 0)
desiredLRP := &models.DesiredLRP{
ProcessGuid: guid,
Domain: "some-domain",
RootFs: "some:rootfs",
Instances: 1,
EnvironmentVariables: []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
CachedDependencies: []*models.CachedDependency{
{Name: "app bits", From: "blobstore.com/bits/app-bits", To: "/usr/local/app", CacheKey: "cache-key", LogSource: "log-source"},
{Name: "app bits with checksum", From: "blobstore.com/bits/app-bits-checksum", To: "/usr/local/app-checksum", CacheKey: "cache-key", LogSource: "log-source", ChecksumAlgorithm: "md5", ChecksumValue: "checksum-value"},
},
Setup: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
StartTimeoutMs: 15000,
Monitor: models.WrapAction(models.EmitProgressFor(
models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
10*time.Second,
),
"start-message",
"success-message",
"failure-message",
)),
DiskMb: 512,
MemoryMb: 1024,
CpuWeight: 42,
Routes: &models.Routes{"my-router": &myRouterJSON},
LogSource: "some-log-source",
LogGuid: "some-log-guid",
MetricsGuid: "some-metrics-guid",
Annotation: "some-annotation",
Network: &models.Network{
Properties: map[string]string{
"some-key": "some-value",
"some-other-key": "some-other-value",
},
},
EgressRules: []*models.SecurityGroupRule{{
Protocol: models.TCPProtocol,
Destinations: []string{"1.1.1.1/32", "2.2.2.2/32"},
PortRange: &models.PortRange{Start: 10, End: 16000},
}},
ModificationTag: &modTag,
LegacyDownloadUser: "******",
TrustedSystemCertificatesPath: "/etc/somepath",
VolumeMounts: []*models.VolumeMount{
{
Driver: "my-driver",
VolumeId: "my-volume",
ContainerPath: "/mnt/mypath",
Mode: models.BindMountMode_RO,
},
},
}
err := desiredLRP.Validate()
Expect(err).NotTo(HaveOccurred())
return desiredLRP
}
func NewValidDesiredLRP(guid string) *models.DesiredLRP {
myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
modTag := models.NewModificationTag("epoch", 0)
desiredLRP := &models.DesiredLRP{
ProcessGuid: guid,
Domain: "some-domain",
RootFs: "some:rootfs",
Instances: 1,
EnvironmentVariables: []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
Setup: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
StartTimeout: 15,
Monitor: models.WrapAction(models.EmitProgressFor(
models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
10*time.Second,
),
"start-message",
"success-message",
"failure-message",
)),
DiskMb: 512,
MemoryMb: 1024,
CpuWeight: 42,
Routes: &models.Routes{"my-router": &myRouterJSON},
LogSource: "some-log-source",
LogGuid: "some-log-guid",
MetricsGuid: "some-metrics-guid",
Annotation: "some-annotation",
EgressRules: []*models.SecurityGroupRule{{
Protocol: models.TCPProtocol,
Destinations: []string{"1.1.1.1/32", "2.2.2.2/32"},
PortRange: &models.PortRange{Start: 10, End: 16000},
}},
ModificationTag: &modTag,
}
err := desiredLRP.Validate()
Expect(err).NotTo(HaveOccurred())
return desiredLRP
}
Context("once the state has been synced with CC", func() {
JustBeforeEach(func() {
Eventually(func() ([]*models.DesiredLRP, error) { return bbsClient.DesiredLRPs(models.DesiredLRPFilter{}) }, 5).Should(HaveLen(3))
})
It("it (adds), (updates), and (removes extra) LRPs", func() {
defaultNofile := recipebuilder.DefaultFileDescriptorLimit
nofile := uint64(16)
expectedSetupActions1 := models.Serial(
&models.DownloadAction{
From: "http://file-server.com/v1/static/some-health-check.tar.gz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
User: "******",
},
&models.DownloadAction{
From: "source-url-1",
To: ".",
CacheKey: "droplets-process-guid-1",
User: "******",
},
)
expectedSetupActions2 := models.Serial(
&models.DownloadAction{
From: "http://file-server.com/v1/static/some-health-check.tar.gz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
User: "******",
},
&models.DownloadAction{
func (backend *traditionalBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
logger.Info("staging-request")
if request.LifecycleData == nil {
return &models.TaskDefinition{}, "", "", ErrMissingLifecycleData
}
var lifecycleData cc_messages.BuildpackStagingData
err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
err = backend.validateRequest(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
compilerURL, err := backend.compilerDownloadURL(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
buildpacksOrder := []string{}
for _, buildpack := range lifecycleData.Buildpacks {
buildpacksOrder = append(buildpacksOrder, buildpack.Key)
}
skipDetect := len(lifecycleData.Buildpacks) == 1 && lifecycleData.Buildpacks[0].SkipDetect
builderConfig := buildpack_app_lifecycle.NewLifecycleBuilderConfig(buildpacksOrder, skipDetect, backend.config.SkipCertVerify)
timeout := traditionalTimeout(request, backend.logger)
actions := []models.ActionInterface{}
//Download app package
appDownloadAction := &models.DownloadAction{
Artifact: "app package",
From: lifecycleData.AppBitsDownloadUri,
To: builderConfig.BuildDir(),
User: "******",
}
actions = append(actions, appDownloadAction)
downloadActions := []models.ActionInterface{}
downloadNames := []string{}
//Download builder
downloadActions = append(
downloadActions,
models.EmitProgressFor(
&models.DownloadAction{
From: compilerURL.String(),
To: path.Dir(builderConfig.ExecutablePath),
CacheKey: fmt.Sprintf("buildpack-%s-lifecycle", lifecycleData.Stack),
User: "******",
},
"",
"",
"Failed to set up staging environment",
),
)
//Download buildpacks
buildpackNames := []string{}
downloadMsgPrefix := ""
if !skipDetect {
downloadMsgPrefix = "No buildpack specified; fetching standard buildpacks to detect and build your application.\n"
}
for _, buildpack := range lifecycleData.Buildpacks {
if buildpack.Name == cc_messages.CUSTOM_BUILDPACK {
buildpackNames = append(buildpackNames, buildpack.Url)
} else {
buildpackNames = append(buildpackNames, buildpack.Name)
downloadActions = append(
downloadActions,
&models.DownloadAction{
Artifact: buildpack.Name,
From: buildpack.Url,
To: builderConfig.BuildpackPath(buildpack.Key),
CacheKey: buildpack.Key,
User: "******",
},
)
}
}
downloadNames = append(downloadNames, fmt.Sprintf("buildpacks (%s)", strings.Join(buildpackNames, ", ")))
//Download buildpack artifacts cache
downloadURL, err := backend.buildArtifactsDownloadURL(lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
if downloadURL != nil {
downloadActions = append(
downloadActions,
models.Try(
&models.DownloadAction{
Artifact: "build artifacts cache",
From: downloadURL.String(),
To: builderConfig.BuildArtifactsCacheDir(),
User: "******",
},
),
)
downloadNames = append(downloadNames, "build artifacts cache")
}
downloadMsg := downloadMsgPrefix + fmt.Sprintf("Downloading %s...", strings.Join(downloadNames, ", "))
actions = append(actions, models.EmitProgressFor(models.Parallel(downloadActions...), downloadMsg, "Downloaded buildpacks", "Downloading buildpacks failed"))
fileDescriptorLimit := uint64(request.FileDescriptors)
//Run Builder
runEnv := append(request.Environment, &models.EnvironmentVariable{"CF_STACK", lifecycleData.Stack})
actions = append(
actions,
models.EmitProgressFor(
&models.RunAction{
User: "******",
Path: builderConfig.Path(),
Args: builderConfig.Args(),
Env: runEnv,
ResourceLimits: &models.ResourceLimits{
Nofile: &fileDescriptorLimit,
},
},
"Staging...",
"Staging complete",
"Staging failed",
),
)
//Upload Droplet
uploadActions := []models.ActionInterface{}
uploadNames := []string{}
uploadURL, err := backend.dropletUploadURL(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
uploadActions = append(
uploadActions,
&models.UploadAction{
Artifact: "droplet",
From: builderConfig.OutputDroplet(), // get the droplet
To: addTimeoutParamToURL(*uploadURL, timeout).String(),
User: "******",
},
)
uploadNames = append(uploadNames, "droplet")
//Upload Buildpack Artifacts Cache
uploadURL, err = backend.buildArtifactsUploadURL(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
uploadActions = append(uploadActions,
models.Try(
&models.UploadAction{
Artifact: "build artifacts cache",
From: builderConfig.OutputBuildArtifactsCache(), // get the compressed build artifacts cache
To: addTimeoutParamToURL(*uploadURL, timeout).String(),
User: "******",
},
),
)
uploadNames = append(uploadNames, "build artifacts cache")
uploadMsg := fmt.Sprintf("Uploading %s...", strings.Join(uploadNames, ", "))
actions = append(actions, models.EmitProgressFor(models.Parallel(uploadActions...), uploadMsg, "Uploading complete", "Uploading failed"))
annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
Lifecycle: TraditionalLifecycleName,
CompletionCallback: request.CompletionCallback,
})
taskDefinition := &models.TaskDefinition{
RootFs: models.PreloadedRootFS(lifecycleData.Stack),
ResultFile: builderConfig.OutputMetadata(),
MemoryMb: int32(request.MemoryMB),
DiskMb: int32(request.DiskMB),
CpuWeight: uint32(StagingTaskCpuWeight),
Action: models.WrapAction(models.Timeout(models.Serial(actions...), timeout)),
LogGuid: request.LogGuid,
LogSource: TaskLogSource,
CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
EgressRules: request.EgressRules,
Annotation: string(annotationJson),
Privileged: true,
EnvironmentVariables: []*models.EnvironmentVariable{{"LANG", DefaultLANG}},
}
logger.Debug("staging-task-request")
return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
func (b *BuildpackRecipeBuilder) Build(desiredApp *cc_messages.DesireAppRequestFromCC) (*models.DesiredLRP, error) {
lrpGuid := desiredApp.ProcessGuid
buildLogger := b.logger.Session("message-builder")
if desiredApp.DropletUri == "" {
buildLogger.Error("desired-app-invalid", ErrDropletSourceMissing, lager.Data{"desired-app": desiredApp})
return nil, ErrDropletSourceMissing
}
if desiredApp.DropletUri != "" && desiredApp.DockerImageUrl != "" {
buildLogger.Error("desired-app-invalid", ErrMultipleAppSources, lager.Data{"desired-app": desiredApp})
return nil, ErrMultipleAppSources
}
var lifecycle = "buildpack/" + desiredApp.Stack
lifecyclePath, ok := b.config.Lifecycles[lifecycle]
if !ok {
buildLogger.Error("unknown-lifecycle", ErrNoLifecycleDefined, lager.Data{
"lifecycle": lifecycle,
})
return nil, ErrNoLifecycleDefined
}
lifecycleURL := lifecycleDownloadURL(lifecyclePath, b.config.FileServerURL)
rootFSPath := models.PreloadedRootFS(desiredApp.Stack)
var containerEnvVars []*models.EnvironmentVariable
containerEnvVars = append(containerEnvVars, &models.EnvironmentVariable{"LANG", DefaultLANG})
numFiles := DefaultFileDescriptorLimit
if desiredApp.FileDescriptors != 0 {
numFiles = desiredApp.FileDescriptors
}
var setup []models.ActionInterface
var actions []models.ActionInterface
var monitor models.ActionInterface
cachedDependencies := []*models.CachedDependency{}
cachedDependencies = append(cachedDependencies, &models.CachedDependency{
From: lifecycleURL,
To: "/tmp/lifecycle",
CacheKey: fmt.Sprintf("%s-lifecycle", strings.Replace(lifecycle, "/", "-", 1)),
})
desiredAppPorts, err := b.ExtractExposedPorts(desiredApp)
if err != nil {
return nil, err
}
switch desiredApp.HealthCheckType {
case cc_messages.PortHealthCheckType, cc_messages.UnspecifiedHealthCheckType:
monitor = models.Timeout(getParallelAction(desiredAppPorts, "vcap"), 30*time.Second)
}
setup = append(setup, &models.DownloadAction{
From: desiredApp.DropletUri,
To: ".",
CacheKey: fmt.Sprintf("droplets-%s", lrpGuid),
User: "******",
})
actions = append(actions, &models.RunAction{
User: "******",
Path: "/tmp/lifecycle/launcher",
Args: append(
[]string{"app"},
desiredApp.StartCommand,
desiredApp.ExecutionMetadata,
),
Env: createLrpEnv(desiredApp.Environment, desiredAppPorts[0]),
LogSource: AppLogSource,
ResourceLimits: &models.ResourceLimits{
Nofile: &numFiles,
},
})
desiredAppRoutingInfo, err := helpers.CCRouteInfoToRoutes(desiredApp.RoutingInfo, desiredAppPorts)
if err != nil {
buildLogger.Error("marshaling-cc-route-info-failed", err)
return nil, err
}
if desiredApp.AllowSSH {
hostKeyPair, err := b.config.KeyFactory.NewKeyPair(1024)
if err != nil {
buildLogger.Error("new-host-key-pair-failed", err)
return nil, err
}
userKeyPair, err := b.config.KeyFactory.NewKeyPair(1024)
if err != nil {
buildLogger.Error("new-user-key-pair-failed", err)
return nil, err
}
actions = append(actions, &models.RunAction{
User: "******",
Path: "/tmp/lifecycle/diego-sshd",
Args: []string{
"-address=" + fmt.Sprintf("0.0.0.0:%d", DefaultSSHPort),
"-hostKey=" + hostKeyPair.PEMEncodedPrivateKey(),
"-authorizedKey=" + userKeyPair.AuthorizedKey(),
"-inheritDaemonEnv",
"-logLevel=fatal",
},
Env: createLrpEnv(desiredApp.Environment, desiredAppPorts[0]),
ResourceLimits: &models.ResourceLimits{
Nofile: &numFiles,
},
})
sshRoutePayload, err := json.Marshal(ssh_routes.SSHRoute{
ContainerPort: 2222,
PrivateKey: userKeyPair.PEMEncodedPrivateKey(),
HostFingerprint: hostKeyPair.Fingerprint(),
})
if err != nil {
buildLogger.Error("marshaling-ssh-route-failed", err)
return nil, err
}
sshRouteMessage := json.RawMessage(sshRoutePayload)
desiredAppRoutingInfo[ssh_routes.DIEGO_SSH] = &sshRouteMessage
desiredAppPorts = append(desiredAppPorts, DefaultSSHPort)
}
setupAction := models.Serial(setup...)
actionAction := models.Codependent(actions...)
return &models.DesiredLRP{
Privileged: true,
Domain: cc_messages.AppLRPDomain,
ProcessGuid: lrpGuid,
Instances: int32(desiredApp.NumInstances),
Routes: &desiredAppRoutingInfo,
Annotation: desiredApp.ETag,
CpuWeight: cpuWeight(desiredApp.MemoryMB),
MemoryMb: int32(desiredApp.MemoryMB),
DiskMb: int32(desiredApp.DiskMB),
Ports: desiredAppPorts,
RootFs: rootFSPath,
LogGuid: desiredApp.LogGuid,
LogSource: LRPLogSource,
MetricsGuid: desiredApp.LogGuid,
EnvironmentVariables: containerEnvVars,
CachedDependencies: cachedDependencies,
Setup: models.WrapAction(setupAction),
Action: models.WrapAction(actionAction),
Monitor: models.WrapAction(monitor),
StartTimeout: uint32(desiredApp.HealthCheckTimeoutInSeconds),
EgressRules: desiredApp.EgressRules,
LegacyDownloadUser: "******",
}, nil
}
func (b *BuildpackRecipeBuilder) BuildTask(task *cc_messages.TaskRequestFromCC) (*models.TaskDefinition, error) {
logger := b.logger.Session("build-task", lager.Data{"request": task})
if task.DropletUri == "" {
logger.Error("missing-droplet-source", ErrDropletSourceMissing)
return nil, ErrDropletSourceMissing
}
if task.DockerPath != "" {
logger.Error("invalid-docker-path", ErrMultipleAppSources)
return nil, ErrMultipleAppSources
}
downloadAction := &models.DownloadAction{
From: task.DropletUri,
To: ".",
CacheKey: "",
User: "******",
}
runAction := &models.RunAction{
User: "******",
Path: "/tmp/lifecycle/launcher",
Args: []string{"app", task.Command, ""},
Env: task.EnvironmentVariables,
LogSource: "TASK",
ResourceLimits: &models.ResourceLimits{},
}
var lifecycle = "buildpack/" + task.RootFs
lifecyclePath, ok := b.config.Lifecycles[lifecycle]
if !ok {
logger.Error("unknown-lifecycle", ErrNoLifecycleDefined, lager.Data{
"lifecycle": lifecycle,
})
return nil, ErrNoLifecycleDefined
}
lifecycleURL := lifecycleDownloadURL(lifecyclePath, b.config.FileServerURL)
cachedDependencies := []*models.CachedDependency{
&models.CachedDependency{
From: lifecycleURL,
To: "/tmp/lifecycle",
CacheKey: fmt.Sprintf("%s-lifecycle", strings.Replace(lifecycle, "/", "-", 1)),
},
}
rootFSPath := models.PreloadedRootFS(task.RootFs)
taskDefinition := &models.TaskDefinition{
Privileged: true,
LogGuid: task.LogGuid,
MemoryMb: int32(task.MemoryMb),
DiskMb: int32(task.DiskMb),
CpuWeight: cpuWeight(task.MemoryMb),
EnvironmentVariables: task.EnvironmentVariables,
RootFs: rootFSPath,
CompletionCallbackUrl: task.CompletionCallbackUrl,
Action: models.WrapAction(models.Serial(
downloadAction,
runAction,
)),
CachedDependencies: cachedDependencies,
EgressRules: task.EgressRules,
LegacyDownloadUser: "******",
}
return taskDefinition, nil
}
"resource_limits": {},
"path": "echo",
"user": "******"
}
}
]
}`,
models.Serial(
&models.Action{
DownloadAction: &models.DownloadAction{
From: "web_location",
To: "local_location",
CacheKey: "elephant",
User: "******",
},
},
&models.Action{
RunAction: &models.RunAction{
Path: "echo",
User: "******",
ResourceLimits: &models.ResourceLimits{},
},
},
),
)
itSerializesAndDeserializes(
`{}`,
models.WrapAction(&models.SerialAction{}),
)
}
defaultNofile := recipebuilder.DefaultFileDescriptorLimit
nofile := uint64(32)
expectedCachedDependencies := []*models.CachedDependency{
{
From: "http://file-server.com/v1/static/some-health-check.tar.gz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
},
}
expectedSetupActions := models.Serial(
&models.DownloadAction{
From: "http://the-droplet.uri.com",
To: ".",
CacheKey: "droplets-the-guid",
User: "******",
},
)
desiredLrpWithoutModificationTag := desiredLrps[0]
desiredLrpWithoutModificationTag.ModificationTag = nil
Expect(desiredLrpWithoutModificationTag).To(Equal(&models.DesiredLRP{
ProcessGuid: "the-guid",
Domain: "cf-apps",
Instances: 3,
RootFs: models.PreloadedRootFS("some-stack"),
StartTimeout: 123456,
EnvironmentVariables: []*models.EnvironmentVariable{
{Name: "LANG", Value: recipebuilder.DefaultLANG},
},
CachedDependencies: expectedCachedDependencies,
Expect(desiredLRP.DiskMb).To(BeEquivalentTo(512))
Expect(desiredLRP.Ports).To(Equal([]uint32{8080}))
Expect(desiredLRP.Privileged).To(BeTrue())
Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))
Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
Expect(desiredLRP.LogSource).To(Equal("CELL"))
Expect(desiredLRP.EnvironmentVariables).To(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))
Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))
expectedSetup := models.Serial(
&models.DownloadAction{
From: "http://the-droplet.uri.com",
To: ".",
CacheKey: "droplets-the-app-guid-the-app-version",
User: "******",
},
)
Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))
expectedCacheDependencies := []*models.CachedDependency{
&models.CachedDependency{
From: "http://file-server.com/v1/static/some-lifecycle.tgz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
},
}
Expect(desiredLRP.CachedDependencies).To(BeEquivalentTo(expectedCacheDependencies))
Expect(desiredLRP.LegacyDownloadUser).To(Equal("vcap"))
Expect(desiredLRP.DiskMb).To(BeEquivalentTo(512))
Expect(desiredLRP.Ports).To(Equal([]uint32{8080}))
Expect(desiredLRP.Privileged).To(BeFalse())
Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))
Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
Expect(desiredLRP.LogSource).To(Equal("CELL"))
Expect(desiredLRP.EnvironmentVariables).NotTo(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))
Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))
expectedSetup := models.Serial(
&models.DownloadAction{
From: "http://file-server.com/v1/static/the/docker/lifecycle/path.tgz",
To: "/tmp/lifecycle",
CacheKey: "docker-lifecycle",
User: "******",
},
)
Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))
parallelRunAction := desiredLRP.Action.CodependentAction
Expect(parallelRunAction.Actions).To(HaveLen(1))
runAction := parallelRunAction.Actions[0].RunAction
Expect(desiredLRP.Monitor.GetValue()).To(Equal(models.Timeout(
&models.ParallelAction{
Actions: []*models.Action{
&models.Action{
RunAction: &models.RunAction{
})) actions := actionsFromTaskDef(taskDef) Expect(actions).To(Equal(models.Serial( downloadAppAction, models.EmitProgressFor( models.Parallel( downloadBuilderAction, downloadFirstBuildpackAction, downloadSecondBuildpackAction, downloadBuildArtifactsAction, ), "No buildpack specified; fetching standard buildpacks to detect and build your application.\n"+ "Downloading buildpacks (zfirst, asecond), build artifacts cache...", "Downloaded buildpacks", "Downloading buildpacks failed", ), runAction, models.EmitProgressFor( models.Parallel( uploadDropletAction, uploadBuildArtifactsAction, ), "Uploading droplet, build artifacts cache...", "Uploading complete", "Uploading failed", ), ).Actions)) Expect(taskDef.MemoryMb).To(Equal(memoryMb)) Expect(taskDef.DiskMb).To(Equal(diskMb))
It("it (adds), (updates), and (removes extra) LRPs", func() {
defaultNofile := recipebuilder.DefaultFileDescriptorLimit
nofile := uint64(16)
expectedCachedDependencies := []*models.CachedDependency{
{
From: "http://file-server.com/v1/static/some-health-check.tar.gz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
},
}
expectedSetupActions1 := models.Serial(
&models.DownloadAction{
From: "source-url-1",
To: ".",
CacheKey: "droplets-process-guid-1",
User: "******",
},
)
expectedSetupActions2 := models.Serial(
&models.DownloadAction{
From: "source-url-1",
To: ".",
CacheKey: "droplets-process-guid-2",
User: "******",
},
)
expectedSetupActions3 := models.Serial(
&models.DownloadAction{
)
BeforeEach(func() {
processGuid = "process-guid-1"
desiredLRP = model_helpers.NewValidDesiredLRP(processGuid)
desiredLRP.DeprecatedStartTimeoutS = 15
desiredLRP.Action = models.WrapAction(&models.TimeoutAction{Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
DeprecatedTimeoutNs: 4 * int64(time.Second),
})
desiredLRP.Setup = models.WrapAction(&models.TimeoutAction{Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
DeprecatedTimeoutNs: 7 * int64(time.Second),
})
desiredLRP.Monitor = models.WrapAction(models.EmitProgressFor(
&models.TimeoutAction{
Action: models.WrapAction(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"})))),
DeprecatedTimeoutNs: 10 * int64(time.Second),
},
"start-message",
"success-message",
"failure-message",
))
})
JustBeforeEach(func() {
schedulingInfo, runInfo := desiredLRP.CreateComponents(fakeClock.Now())
runInfo.DeprecatedStartTimeoutS = 15
_, err := json.Marshal(desiredLRP.Routes)
Expect(err).NotTo(HaveOccurred())
})
var newValidDesiredLRP = func(guid string) *models.DesiredLRP {
myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
modTag := models.NewModificationTag("epoch", 0)
desiredLRP := &models.DesiredLRP{
ProcessGuid: guid,
Domain: "some-domain",
RootFs: "some:rootfs",
Instances: 1,
EnvironmentVariables: []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
Setup: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
DeprecatedStartTimeoutS: 15,
Monitor: models.WrapAction(models.EmitProgressFor(
models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
10*time.Second,
),
"start-message",
"success-message",
"failure-message",
)),
DiskMb: 512,
MemoryMb: 1024,
CpuWeight: 42,
Routes: &models.Routes{"my-router": &myRouterJSON},
LogSource: "some-log-source",
LogGuid: "some-log-guid",
MetricsGuid: "some-metrics-guid",
Annotation: "some-annotation",
EgressRules: []*models.SecurityGroupRule{{
BeforeEach(func() {
newlyDesiredTask = &models.TaskDefinition{
LogGuid: "some-log-guid",
MemoryMb: 128,
DiskMb: 512,
EnvironmentVariables: []*models.EnvironmentVariable{
{Name: "foo", Value: "bar"},
{Name: "VCAP_APPLICATION", Value: "{\"application_name\":\"my-app\"}"},
},
RootFs: "http://docker-image.com",
CompletionCallbackUrl: "http://api.cc.com/v1/tasks/complete",
Action: models.WrapAction(models.Serial(
&models.DownloadAction{
From: taskRequest.DropletUri,
To: ".",
CacheKey: "",
User: "******",
},
&models.RunAction{},
)),
}
buildpackBuilder.BuildTaskReturns(newlyDesiredTask, nil)
})
It("logs the incoming and outgoing request", func() {
Eventually(logger.TestSink.Buffer).Should(gbytes.Say("serving"))
Eventually(logger.TestSink.Buffer).Should(gbytes.Say("desiring-task"))
})
It("creates the task", func() {
"run": {
"resource_limits": {},
"path": "echo",
"user": "******",
"suppress_log_output": false
}
}
]
}`,
models.WrapAction(models.Serial(
&models.DownloadAction{
From: "web_location",
To: "local_location",
CacheKey: "elephant",
User: "******",
},
&models.RunAction{
Path: "echo",
User: "******",
ResourceLimits: &models.ResourceLimits{},
},
)),
)
Describe("Validate", func() {
var serialAction *models.SerialAction
Context("when the action has 'actions' as a slice of valid actions", func() {
It("is valid", func() {
serialAction = models.Serial(
&models.UploadAction{
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
logger.Info("staging-request")
var lifecycleData cc_messages.DockerStagingData
err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
err = backend.validateRequest(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
compilerURL, err := backend.compilerDownloadURL()
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
cacheDockerImage := false
for _, envVar := range request.Environment {
if envVar.Name == "DIEGO_DOCKER_CACHE" && envVar.Value == "true" {
cacheDockerImage = true
break
}
}
actions := []models.ActionInterface{}
//Download builder
actions = append(
actions,
models.EmitProgressFor(
&models.DownloadAction{
From: compilerURL.String(),
To: path.Dir(DockerBuilderExecutablePath),
CacheKey: "docker-lifecycle",
User: "******",
},
"",
"",
"Failed to set up docker environment",
),
)
runActionArguments := []string{"-outputMetadataJSONFilename", DockerBuilderOutputPath, "-dockerRef", lifecycleData.DockerImageUrl}
runAs := "vcap"
if cacheDockerImage {
runAs = "root"
host, port, err := net.SplitHostPort(backend.config.DockerRegistryAddress)
if err != nil {
logger.Debug("invalid docker registry address", lager.Data{"address": backend.config.DockerRegistryAddress, "error": err.Error()})
return &models.TaskDefinition{}, "", "", ErrInvalidDockerRegistryAddress
}
registryServices, err := getDockerRegistryServices(backend.config.ConsulCluster, backend.logger)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
registryRules := addDockerRegistryRules(request.EgressRules, registryServices)
request.EgressRules = append(request.EgressRules, registryRules...)
registryIPs := strings.Join(buildDockerRegistryAddresses(registryServices), ",")
runActionArguments, err = addDockerCachingArguments(runActionArguments, registryIPs, backend.config.InsecureDockerRegistry, host, port, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
}
fileDescriptorLimit := uint64(request.FileDescriptors)
// Run builder
actions = append(
actions,
models.EmitProgressFor(
&models.RunAction{
Path: DockerBuilderExecutablePath,
Args: runActionArguments,
Env: request.Environment,
ResourceLimits: &models.ResourceLimits{
Nofile: &fileDescriptorLimit,
},
User: runAs,
},
"Staging...",
"Staging Complete",
"Staging Failed",
),
)
annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
Lifecycle: DockerLifecycleName,
})
taskDefinition := &models.TaskDefinition{
RootFs: models.PreloadedRootFS(backend.config.DockerStagingStack),
ResultFile: DockerBuilderOutputPath,
Privileged: true,
MemoryMb: int32(request.MemoryMB),
LogSource: TaskLogSource,
LogGuid: request.LogGuid,
EgressRules: request.EgressRules,
DiskMb: int32(request.DiskMB),
CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
Annotation: string(annotationJson),
Action: models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
}
logger.Debug("staging-task-request")
return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
Expect(task.Failed).To(BeFalse())
})
Context("when the command exceeds its memory limit", func() {
It("should fail the Task", func() {
err := receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
guid,
models.Serial(
&models.RunAction{
User: "******",
Path: "curl",
Args: []string{inigo_announcement_server.AnnounceURL("before-memory-overdose")},
},
&models.RunAction{
User: "******",
Path: "sh",
Args: []string{"-c", "yes $(yes)"},
},
&models.RunAction{
User: "******",
Path: "curl",
Args: []string{inigo_announcement_server.AnnounceURL("after-memory-overdose")},
},
),
10,
1024,
))
Expect(err).NotTo(HaveOccurred())
Eventually(inigo_announcement_server.Announcements).Should(ContainElement("before-memory-overdose"))
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
logger.Info("staging-request")
var lifecycleData cc_messages.DockerStagingData
err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
err = backend.validateRequest(request, lifecycleData)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
compilerURL, err := backend.compilerDownloadURL()
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
cachedDependencies := []*models.CachedDependency{
&models.CachedDependency{
From: compilerURL.String(),
To: path.Dir(DockerBuilderExecutablePath),
CacheKey: "docker-lifecycle",
},
}
runActionArguments := []string{
"-outputMetadataJSONFilename", DockerBuilderOutputPath,
"-dockerRef", lifecycleData.DockerImageUrl,
}
if len(backend.config.InsecureDockerRegistries) > 0 {
insecureDockerRegistries := strings.Join(backend.config.InsecureDockerRegistries, ",")
runActionArguments = append(runActionArguments, "-insecureDockerRegistries", insecureDockerRegistries)
}
fileDescriptorLimit := uint64(request.FileDescriptors)
runAs := "vcap"
actions := []models.ActionInterface{}
if cacheDockerImage(request.Environment) {
runAs = "root"
additionalEgressRules, additionalArgs, err := cachingEgressRulesAndArgs(
logger,
backend.config.DockerRegistryAddress,
backend.config.ConsulCluster,
lifecycleData,
)
if err != nil {
return &models.TaskDefinition{}, "", "", err
}
runActionArguments = append(runActionArguments, additionalArgs...)
request.EgressRules = append(request.EgressRules, additionalEgressRules...)
actions = append(
actions,
models.EmitProgressFor(
&models.RunAction{
Path: MountCgroupsPath,
ResourceLimits: &models.ResourceLimits{
Nofile: &fileDescriptorLimit,
},
User: runAs,
},
"Preparing docker daemon...",
"",
"Failed to set up docker environment",
),
)
}
actions = append(
actions,
models.EmitProgressFor(
&models.RunAction{
Path: DockerBuilderExecutablePath,
Args: runActionArguments,
Env: request.Environment,
ResourceLimits: &models.ResourceLimits{
Nofile: &fileDescriptorLimit,
},
User: runAs,
},
"Staging...",
"Staging Complete",
"Staging Failed",
),
)
annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
Lifecycle: DockerLifecycleName,
CompletionCallback: request.CompletionCallback,
})
taskDefinition := &models.TaskDefinition{
RootFs: models.PreloadedRootFS(backend.config.DockerStagingStack),
ResultFile: DockerBuilderOutputPath,
Privileged: true,
MemoryMb: int32(request.MemoryMB),
LogSource: TaskLogSource,
LogGuid: request.LogGuid,
EgressRules: request.EgressRules,
DiskMb: int32(request.DiskMB),
CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
Annotation: string(annotationJson),
Action: models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
CachedDependencies: cachedDependencies,
LegacyDownloadUser: "******",
TrustedSystemCertificatesPath: TrustedSystemCertificatesPath,
}
logger.Debug("staging-task-request")
return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))
Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
Expect(desiredLRP.LogSource).To(Equal("CELL"))
Expect(desiredLRP.EnvironmentVariables).To(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))
Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))
expectedSetup := models.Serial(
&models.DownloadAction{
From: "http://file-server.com/v1/static/some-lifecycle.tgz",
To: "/tmp/lifecycle",
CacheKey: "buildpack-some-stack-lifecycle",
User: "******",
},
&models.DownloadAction{
From: "http://the-droplet.uri.com",
To: ".",
CacheKey: "droplets-the-app-guid-the-app-version",
User: "******",
},
)
Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))
parallelRunAction := desiredLRP.Action.CodependentAction
Expect(parallelRunAction.Actions).To(HaveLen(1))
runAction := parallelRunAction.Actions[0].RunAction
Expect(desiredLRP.Monitor.GetValue()).To(Equal(models.Timeout(
&models.ParallelAction{