Example #1
0
func NewValidDesiredLRP(guid string) *models.DesiredLRP {
	myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
	modTag := models.NewModificationTag("epoch", 0)
	desiredLRP := &models.DesiredLRP{
		ProcessGuid:          guid,
		Domain:               "some-domain",
		RootFs:               "some:rootfs",
		Instances:            1,
		EnvironmentVariables: []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
		CachedDependencies: []*models.CachedDependency{
			{Name: "app bits", From: "blobstore.com/bits/app-bits", To: "/usr/local/app", CacheKey: "cache-key", LogSource: "log-source"},
			{Name: "app bits with checksum", From: "blobstore.com/bits/app-bits-checksum", To: "/usr/local/app-checksum", CacheKey: "cache-key", LogSource: "log-source", ChecksumAlgorithm: "md5", ChecksumValue: "checksum-value"},
		},
		Setup:          models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
		Action:         models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
		StartTimeoutMs: 15000,
		Monitor: models.WrapAction(models.EmitProgressFor(
			models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
				10*time.Second,
			),
			"start-message",
			"success-message",
			"failure-message",
		)),
		DiskMb:      512,
		MemoryMb:    1024,
		CpuWeight:   42,
		Routes:      &models.Routes{"my-router": &myRouterJSON},
		LogSource:   "some-log-source",
		LogGuid:     "some-log-guid",
		MetricsGuid: "some-metrics-guid",
		Annotation:  "some-annotation",
		Network: &models.Network{
			Properties: map[string]string{
				"some-key":       "some-value",
				"some-other-key": "some-other-value",
			},
		},
		EgressRules: []*models.SecurityGroupRule{{
			Protocol:     models.TCPProtocol,
			Destinations: []string{"1.1.1.1/32", "2.2.2.2/32"},
			PortRange:    &models.PortRange{Start: 10, End: 16000},
		}},
		ModificationTag:               &modTag,
		LegacyDownloadUser:            "******",
		TrustedSystemCertificatesPath: "/etc/somepath",
		VolumeMounts: []*models.VolumeMount{
			{
				Driver:        "my-driver",
				VolumeId:      "my-volume",
				ContainerPath: "/mnt/mypath",
				Mode:          models.BindMountMode_RO,
			},
		},
	}
	err := desiredLRP.Validate()
	Expect(err).NotTo(HaveOccurred())

	return desiredLRP
}
Example #2
0
func NewValidDesiredLRP(guid string) *models.DesiredLRP {
	myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
	modTag := models.NewModificationTag("epoch", 0)
	desiredLRP := &models.DesiredLRP{
		ProcessGuid:          guid,
		Domain:               "some-domain",
		RootFs:               "some:rootfs",
		Instances:            1,
		EnvironmentVariables: []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
		Setup:                models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
		Action:               models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
		StartTimeout:         15,
		Monitor: models.WrapAction(models.EmitProgressFor(
			models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
				10*time.Second,
			),
			"start-message",
			"success-message",
			"failure-message",
		)),
		DiskMb:      512,
		MemoryMb:    1024,
		CpuWeight:   42,
		Routes:      &models.Routes{"my-router": &myRouterJSON},
		LogSource:   "some-log-source",
		LogGuid:     "some-log-guid",
		MetricsGuid: "some-metrics-guid",
		Annotation:  "some-annotation",
		EgressRules: []*models.SecurityGroupRule{{
			Protocol:     models.TCPProtocol,
			Destinations: []string{"1.1.1.1/32", "2.2.2.2/32"},
			PortRange:    &models.PortRange{Start: 10, End: 16000},
		}},
		ModificationTag: &modTag,
	}
	err := desiredLRP.Validate()
	Expect(err).NotTo(HaveOccurred())

	return desiredLRP
}
Example #3
0
		Context("once the state has been synced with CC", func() {
			JustBeforeEach(func() {
				Eventually(func() ([]*models.DesiredLRP, error) { return bbsClient.DesiredLRPs(models.DesiredLRPFilter{}) }, 5).Should(HaveLen(3))
			})

			It("it (adds), (updates), and (removes extra) LRPs", func() {
				defaultNofile := recipebuilder.DefaultFileDescriptorLimit
				nofile := uint64(16)

				expectedSetupActions1 := models.Serial(
					&models.DownloadAction{
						From:     "http://file-server.com/v1/static/some-health-check.tar.gz",
						To:       "/tmp/lifecycle",
						CacheKey: "buildpack-some-stack-lifecycle",
						User:     "******",
					},
					&models.DownloadAction{
						From:     "source-url-1",
						To:       ".",
						CacheKey: "droplets-process-guid-1",
						User:     "******",
					},
				)

				expectedSetupActions2 := models.Serial(
					&models.DownloadAction{
						From:     "http://file-server.com/v1/static/some-health-check.tar.gz",
						To:       "/tmp/lifecycle",
						CacheKey: "buildpack-some-stack-lifecycle",
						User:     "******",
					},
					&models.DownloadAction{
func (backend *traditionalBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
	logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
	logger.Info("staging-request")

	if request.LifecycleData == nil {
		return &models.TaskDefinition{}, "", "", ErrMissingLifecycleData
	}

	var lifecycleData cc_messages.BuildpackStagingData
	err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	err = backend.validateRequest(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	compilerURL, err := backend.compilerDownloadURL(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	buildpacksOrder := []string{}
	for _, buildpack := range lifecycleData.Buildpacks {
		buildpacksOrder = append(buildpacksOrder, buildpack.Key)
	}

	skipDetect := len(lifecycleData.Buildpacks) == 1 && lifecycleData.Buildpacks[0].SkipDetect

	builderConfig := buildpack_app_lifecycle.NewLifecycleBuilderConfig(buildpacksOrder, skipDetect, backend.config.SkipCertVerify)

	timeout := traditionalTimeout(request, backend.logger)

	actions := []models.ActionInterface{}

	//Download app package
	appDownloadAction := &models.DownloadAction{
		Artifact: "app package",
		From:     lifecycleData.AppBitsDownloadUri,
		To:       builderConfig.BuildDir(),
		User:     "******",
	}

	actions = append(actions, appDownloadAction)

	downloadActions := []models.ActionInterface{}
	downloadNames := []string{}

	//Download builder
	downloadActions = append(
		downloadActions,
		models.EmitProgressFor(
			&models.DownloadAction{
				From:     compilerURL.String(),
				To:       path.Dir(builderConfig.ExecutablePath),
				CacheKey: fmt.Sprintf("buildpack-%s-lifecycle", lifecycleData.Stack),
				User:     "******",
			},
			"",
			"",
			"Failed to set up staging environment",
		),
	)

	//Download buildpacks
	buildpackNames := []string{}
	downloadMsgPrefix := ""
	if !skipDetect {
		downloadMsgPrefix = "No buildpack specified; fetching standard buildpacks to detect and build your application.\n"
	}
	for _, buildpack := range lifecycleData.Buildpacks {
		if buildpack.Name == cc_messages.CUSTOM_BUILDPACK {
			buildpackNames = append(buildpackNames, buildpack.Url)
		} else {
			buildpackNames = append(buildpackNames, buildpack.Name)
			downloadActions = append(
				downloadActions,
				&models.DownloadAction{
					Artifact: buildpack.Name,
					From:     buildpack.Url,
					To:       builderConfig.BuildpackPath(buildpack.Key),
					CacheKey: buildpack.Key,
					User:     "******",
				},
			)
		}
	}

	downloadNames = append(downloadNames, fmt.Sprintf("buildpacks (%s)", strings.Join(buildpackNames, ", ")))

	//Download buildpack artifacts cache
	downloadURL, err := backend.buildArtifactsDownloadURL(lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	if downloadURL != nil {
		downloadActions = append(
			downloadActions,
			models.Try(
				&models.DownloadAction{
					Artifact: "build artifacts cache",
					From:     downloadURL.String(),
					To:       builderConfig.BuildArtifactsCacheDir(),
					User:     "******",
				},
			),
		)
		downloadNames = append(downloadNames, "build artifacts cache")
	}

	downloadMsg := downloadMsgPrefix + fmt.Sprintf("Downloading %s...", strings.Join(downloadNames, ", "))
	actions = append(actions, models.EmitProgressFor(models.Parallel(downloadActions...), downloadMsg, "Downloaded buildpacks", "Downloading buildpacks failed"))

	fileDescriptorLimit := uint64(request.FileDescriptors)

	//Run Builder
	runEnv := append(request.Environment, &models.EnvironmentVariable{"CF_STACK", lifecycleData.Stack})
	actions = append(
		actions,
		models.EmitProgressFor(
			&models.RunAction{
				User: "******",
				Path: builderConfig.Path(),
				Args: builderConfig.Args(),
				Env:  runEnv,
				ResourceLimits: &models.ResourceLimits{
					Nofile: &fileDescriptorLimit,
				},
			},
			"Staging...",
			"Staging complete",
			"Staging failed",
		),
	)

	//Upload Droplet
	uploadActions := []models.ActionInterface{}
	uploadNames := []string{}
	uploadURL, err := backend.dropletUploadURL(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	uploadActions = append(
		uploadActions,
		&models.UploadAction{
			Artifact: "droplet",
			From:     builderConfig.OutputDroplet(), // get the droplet
			To:       addTimeoutParamToURL(*uploadURL, timeout).String(),
			User:     "******",
		},
	)
	uploadNames = append(uploadNames, "droplet")

	//Upload Buildpack Artifacts Cache
	uploadURL, err = backend.buildArtifactsUploadURL(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	uploadActions = append(uploadActions,
		models.Try(
			&models.UploadAction{
				Artifact: "build artifacts cache",
				From:     builderConfig.OutputBuildArtifactsCache(), // get the compressed build artifacts cache
				To:       addTimeoutParamToURL(*uploadURL, timeout).String(),
				User:     "******",
			},
		),
	)
	uploadNames = append(uploadNames, "build artifacts cache")

	uploadMsg := fmt.Sprintf("Uploading %s...", strings.Join(uploadNames, ", "))
	actions = append(actions, models.EmitProgressFor(models.Parallel(uploadActions...), uploadMsg, "Uploading complete", "Uploading failed"))

	annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
		Lifecycle:          TraditionalLifecycleName,
		CompletionCallback: request.CompletionCallback,
	})

	taskDefinition := &models.TaskDefinition{
		RootFs:                models.PreloadedRootFS(lifecycleData.Stack),
		ResultFile:            builderConfig.OutputMetadata(),
		MemoryMb:              int32(request.MemoryMB),
		DiskMb:                int32(request.DiskMB),
		CpuWeight:             uint32(StagingTaskCpuWeight),
		Action:                models.WrapAction(models.Timeout(models.Serial(actions...), timeout)),
		LogGuid:               request.LogGuid,
		LogSource:             TaskLogSource,
		CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
		EgressRules:           request.EgressRules,
		Annotation:            string(annotationJson),
		Privileged:            true,
		EnvironmentVariables:  []*models.EnvironmentVariable{{"LANG", DefaultLANG}},
	}

	logger.Debug("staging-task-request")

	return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
func (b *BuildpackRecipeBuilder) Build(desiredApp *cc_messages.DesireAppRequestFromCC) (*models.DesiredLRP, error) {
	lrpGuid := desiredApp.ProcessGuid

	buildLogger := b.logger.Session("message-builder")

	if desiredApp.DropletUri == "" {
		buildLogger.Error("desired-app-invalid", ErrDropletSourceMissing, lager.Data{"desired-app": desiredApp})
		return nil, ErrDropletSourceMissing
	}

	if desiredApp.DropletUri != "" && desiredApp.DockerImageUrl != "" {
		buildLogger.Error("desired-app-invalid", ErrMultipleAppSources, lager.Data{"desired-app": desiredApp})
		return nil, ErrMultipleAppSources
	}

	var lifecycle = "buildpack/" + desiredApp.Stack
	lifecyclePath, ok := b.config.Lifecycles[lifecycle]
	if !ok {
		buildLogger.Error("unknown-lifecycle", ErrNoLifecycleDefined, lager.Data{
			"lifecycle": lifecycle,
		})

		return nil, ErrNoLifecycleDefined
	}

	lifecycleURL := lifecycleDownloadURL(lifecyclePath, b.config.FileServerURL)

	rootFSPath := models.PreloadedRootFS(desiredApp.Stack)

	var containerEnvVars []*models.EnvironmentVariable
	containerEnvVars = append(containerEnvVars, &models.EnvironmentVariable{"LANG", DefaultLANG})

	numFiles := DefaultFileDescriptorLimit
	if desiredApp.FileDescriptors != 0 {
		numFiles = desiredApp.FileDescriptors
	}

	var setup []models.ActionInterface
	var actions []models.ActionInterface
	var monitor models.ActionInterface

	cachedDependencies := []*models.CachedDependency{}
	cachedDependencies = append(cachedDependencies, &models.CachedDependency{
		From:     lifecycleURL,
		To:       "/tmp/lifecycle",
		CacheKey: fmt.Sprintf("%s-lifecycle", strings.Replace(lifecycle, "/", "-", 1)),
	})

	desiredAppPorts, err := b.ExtractExposedPorts(desiredApp)
	if err != nil {
		return nil, err
	}

	switch desiredApp.HealthCheckType {
	case cc_messages.PortHealthCheckType, cc_messages.UnspecifiedHealthCheckType:
		monitor = models.Timeout(getParallelAction(desiredAppPorts, "vcap"), 30*time.Second)
	}

	setup = append(setup, &models.DownloadAction{
		From:     desiredApp.DropletUri,
		To:       ".",
		CacheKey: fmt.Sprintf("droplets-%s", lrpGuid),
		User:     "******",
	})

	actions = append(actions, &models.RunAction{
		User: "******",
		Path: "/tmp/lifecycle/launcher",
		Args: append(
			[]string{"app"},
			desiredApp.StartCommand,
			desiredApp.ExecutionMetadata,
		),
		Env:       createLrpEnv(desiredApp.Environment, desiredAppPorts[0]),
		LogSource: AppLogSource,
		ResourceLimits: &models.ResourceLimits{
			Nofile: &numFiles,
		},
	})

	desiredAppRoutingInfo, err := helpers.CCRouteInfoToRoutes(desiredApp.RoutingInfo, desiredAppPorts)
	if err != nil {
		buildLogger.Error("marshaling-cc-route-info-failed", err)
		return nil, err
	}

	if desiredApp.AllowSSH {
		hostKeyPair, err := b.config.KeyFactory.NewKeyPair(1024)
		if err != nil {
			buildLogger.Error("new-host-key-pair-failed", err)
			return nil, err
		}

		userKeyPair, err := b.config.KeyFactory.NewKeyPair(1024)
		if err != nil {
			buildLogger.Error("new-user-key-pair-failed", err)
			return nil, err
		}

		actions = append(actions, &models.RunAction{
			User: "******",
			Path: "/tmp/lifecycle/diego-sshd",
			Args: []string{
				"-address=" + fmt.Sprintf("0.0.0.0:%d", DefaultSSHPort),
				"-hostKey=" + hostKeyPair.PEMEncodedPrivateKey(),
				"-authorizedKey=" + userKeyPair.AuthorizedKey(),
				"-inheritDaemonEnv",
				"-logLevel=fatal",
			},
			Env: createLrpEnv(desiredApp.Environment, desiredAppPorts[0]),
			ResourceLimits: &models.ResourceLimits{
				Nofile: &numFiles,
			},
		})

		sshRoutePayload, err := json.Marshal(ssh_routes.SSHRoute{
			ContainerPort:   2222,
			PrivateKey:      userKeyPair.PEMEncodedPrivateKey(),
			HostFingerprint: hostKeyPair.Fingerprint(),
		})

		if err != nil {
			buildLogger.Error("marshaling-ssh-route-failed", err)
			return nil, err
		}

		sshRouteMessage := json.RawMessage(sshRoutePayload)
		desiredAppRoutingInfo[ssh_routes.DIEGO_SSH] = &sshRouteMessage
		desiredAppPorts = append(desiredAppPorts, DefaultSSHPort)
	}

	setupAction := models.Serial(setup...)
	actionAction := models.Codependent(actions...)

	return &models.DesiredLRP{
		Privileged: true,

		Domain: cc_messages.AppLRPDomain,

		ProcessGuid: lrpGuid,
		Instances:   int32(desiredApp.NumInstances),
		Routes:      &desiredAppRoutingInfo,
		Annotation:  desiredApp.ETag,

		CpuWeight: cpuWeight(desiredApp.MemoryMB),

		MemoryMb: int32(desiredApp.MemoryMB),
		DiskMb:   int32(desiredApp.DiskMB),

		Ports: desiredAppPorts,

		RootFs: rootFSPath,

		LogGuid:   desiredApp.LogGuid,
		LogSource: LRPLogSource,

		MetricsGuid: desiredApp.LogGuid,

		EnvironmentVariables: containerEnvVars,
		CachedDependencies:   cachedDependencies,
		Setup:                models.WrapAction(setupAction),
		Action:               models.WrapAction(actionAction),
		Monitor:              models.WrapAction(monitor),

		StartTimeout: uint32(desiredApp.HealthCheckTimeoutInSeconds),

		EgressRules:        desiredApp.EgressRules,
		LegacyDownloadUser: "******",
	}, nil
}
func (b *BuildpackRecipeBuilder) BuildTask(task *cc_messages.TaskRequestFromCC) (*models.TaskDefinition, error) {
	logger := b.logger.Session("build-task", lager.Data{"request": task})

	if task.DropletUri == "" {
		logger.Error("missing-droplet-source", ErrDropletSourceMissing)
		return nil, ErrDropletSourceMissing
	}

	if task.DockerPath != "" {
		logger.Error("invalid-docker-path", ErrMultipleAppSources)
		return nil, ErrMultipleAppSources
	}

	downloadAction := &models.DownloadAction{
		From:     task.DropletUri,
		To:       ".",
		CacheKey: "",
		User:     "******",
	}

	runAction := &models.RunAction{
		User:           "******",
		Path:           "/tmp/lifecycle/launcher",
		Args:           []string{"app", task.Command, ""},
		Env:            task.EnvironmentVariables,
		LogSource:      "TASK",
		ResourceLimits: &models.ResourceLimits{},
	}

	var lifecycle = "buildpack/" + task.RootFs
	lifecyclePath, ok := b.config.Lifecycles[lifecycle]
	if !ok {
		logger.Error("unknown-lifecycle", ErrNoLifecycleDefined, lager.Data{
			"lifecycle": lifecycle,
		})

		return nil, ErrNoLifecycleDefined
	}

	lifecycleURL := lifecycleDownloadURL(lifecyclePath, b.config.FileServerURL)

	cachedDependencies := []*models.CachedDependency{
		&models.CachedDependency{
			From:     lifecycleURL,
			To:       "/tmp/lifecycle",
			CacheKey: fmt.Sprintf("%s-lifecycle", strings.Replace(lifecycle, "/", "-", 1)),
		},
	}

	rootFSPath := models.PreloadedRootFS(task.RootFs)

	taskDefinition := &models.TaskDefinition{
		Privileged:            true,
		LogGuid:               task.LogGuid,
		MemoryMb:              int32(task.MemoryMb),
		DiskMb:                int32(task.DiskMb),
		CpuWeight:             cpuWeight(task.MemoryMb),
		EnvironmentVariables:  task.EnvironmentVariables,
		RootFs:                rootFSPath,
		CompletionCallbackUrl: task.CompletionCallbackUrl,
		Action: models.WrapAction(models.Serial(
			downloadAction,
			runAction,
		)),
		CachedDependencies: cachedDependencies,
		EgressRules:        task.EgressRules,
		LegacyDownloadUser: "******",
	}

	return taskDefinition, nil
}
Example #7
0
								"resource_limits": {},
								"path": "echo",
								"user": "******"
							}
						}
					]
			}`,
			models.Serial(
				&models.Action{
					DownloadAction: &models.DownloadAction{
						From:     "web_location",
						To:       "local_location",
						CacheKey: "elephant",
						User:     "******",
					},
				},
				&models.Action{
					RunAction: &models.RunAction{
						Path:           "echo",
						User:           "******",
						ResourceLimits: &models.ResourceLimits{},
					},
				},
			),
		)

		itSerializesAndDeserializes(
			`{}`,
			models.WrapAction(&models.SerialAction{}),
		)
Example #8
0
			}
			defaultNofile := recipebuilder.DefaultFileDescriptorLimit
			nofile := uint64(32)

			expectedCachedDependencies := []*models.CachedDependency{
				{
					From:     "http://file-server.com/v1/static/some-health-check.tar.gz",
					To:       "/tmp/lifecycle",
					CacheKey: "buildpack-some-stack-lifecycle",
				},
			}

			expectedSetupActions := models.Serial(
				&models.DownloadAction{
					From:     "http://the-droplet.uri.com",
					To:       ".",
					CacheKey: "droplets-the-guid",
					User:     "******",
				},
			)
			desiredLrpWithoutModificationTag := desiredLrps[0]
			desiredLrpWithoutModificationTag.ModificationTag = nil
			Expect(desiredLrpWithoutModificationTag).To(Equal(&models.DesiredLRP{
				ProcessGuid:  "the-guid",
				Domain:       "cf-apps",
				Instances:    3,
				RootFs:       models.PreloadedRootFS("some-stack"),
				StartTimeout: 123456,
				EnvironmentVariables: []*models.EnvironmentVariable{
					{Name: "LANG", Value: recipebuilder.DefaultLANG},
				},
				CachedDependencies: expectedCachedDependencies,
				Expect(desiredLRP.DiskMb).To(BeEquivalentTo(512))
				Expect(desiredLRP.Ports).To(Equal([]uint32{8080}))
				Expect(desiredLRP.Privileged).To(BeTrue())
				Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))

				Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
				Expect(desiredLRP.LogSource).To(Equal("CELL"))

				Expect(desiredLRP.EnvironmentVariables).To(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))

				Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))

				expectedSetup := models.Serial(
					&models.DownloadAction{
						From:     "http://the-droplet.uri.com",
						To:       ".",
						CacheKey: "droplets-the-app-guid-the-app-version",
						User:     "******",
					},
				)
				Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))

				expectedCacheDependencies := []*models.CachedDependency{
					&models.CachedDependency{
						From:     "http://file-server.com/v1/static/some-lifecycle.tgz",
						To:       "/tmp/lifecycle",
						CacheKey: "buildpack-some-stack-lifecycle",
					},
				}
				Expect(desiredLRP.CachedDependencies).To(BeEquivalentTo(expectedCacheDependencies))
				Expect(desiredLRP.LegacyDownloadUser).To(Equal("vcap"))
			Expect(desiredLRP.DiskMb).To(BeEquivalentTo(512))
			Expect(desiredLRP.Ports).To(Equal([]uint32{8080}))
			Expect(desiredLRP.Privileged).To(BeFalse())
			Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))

			Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
			Expect(desiredLRP.LogSource).To(Equal("CELL"))

			Expect(desiredLRP.EnvironmentVariables).NotTo(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))

			Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))

			expectedSetup := models.Serial(
				&models.DownloadAction{
					From:     "http://file-server.com/v1/static/the/docker/lifecycle/path.tgz",
					To:       "/tmp/lifecycle",
					CacheKey: "docker-lifecycle",
					User:     "******",
				},
			)
			Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))

			parallelRunAction := desiredLRP.Action.CodependentAction
			Expect(parallelRunAction.Actions).To(HaveLen(1))

			runAction := parallelRunAction.Actions[0].RunAction

			Expect(desiredLRP.Monitor.GetValue()).To(Equal(models.Timeout(
				&models.ParallelAction{
					Actions: []*models.Action{
						&models.Action{
							RunAction: &models.RunAction{
		}))

		actions := actionsFromTaskDef(taskDef)
		Expect(actions).To(Equal(models.Serial(
			downloadAppAction,
			models.EmitProgressFor(
				models.Parallel(
					downloadBuilderAction,
					downloadFirstBuildpackAction,
					downloadSecondBuildpackAction,
					downloadBuildArtifactsAction,
				),
				"No buildpack specified; fetching standard buildpacks to detect and build your application.\n"+
					"Downloading buildpacks (zfirst, asecond), build artifacts cache...",
				"Downloaded buildpacks",
				"Downloading buildpacks failed",
			),
			runAction,
			models.EmitProgressFor(
				models.Parallel(
					uploadDropletAction,
					uploadBuildArtifactsAction,
				),
				"Uploading droplet, build artifacts cache...",
				"Uploading complete",
				"Uploading failed",
			),
		).Actions))

		Expect(taskDef.MemoryMb).To(Equal(memoryMb))
		Expect(taskDef.DiskMb).To(Equal(diskMb))
Example #12
0
			It("it (adds), (updates), and (removes extra) LRPs", func() {
				defaultNofile := recipebuilder.DefaultFileDescriptorLimit
				nofile := uint64(16)

				expectedCachedDependencies := []*models.CachedDependency{
					{
						From:     "http://file-server.com/v1/static/some-health-check.tar.gz",
						To:       "/tmp/lifecycle",
						CacheKey: "buildpack-some-stack-lifecycle",
					},
				}

				expectedSetupActions1 := models.Serial(
					&models.DownloadAction{
						From:     "source-url-1",
						To:       ".",
						CacheKey: "droplets-process-guid-1",
						User:     "******",
					},
				)

				expectedSetupActions2 := models.Serial(
					&models.DownloadAction{
						From:     "source-url-1",
						To:       ".",
						CacheKey: "droplets-process-guid-2",
						User:     "******",
					},
				)

				expectedSetupActions3 := models.Serial(
					&models.DownloadAction{
			)

			BeforeEach(func() {
				processGuid = "process-guid-1"
				desiredLRP = model_helpers.NewValidDesiredLRP(processGuid)
				desiredLRP.DeprecatedStartTimeoutS = 15
				desiredLRP.Action = models.WrapAction(&models.TimeoutAction{Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
					DeprecatedTimeoutNs: 4 * int64(time.Second),
				})

				desiredLRP.Setup = models.WrapAction(&models.TimeoutAction{Action: models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
					DeprecatedTimeoutNs: 7 * int64(time.Second),
				})
				desiredLRP.Monitor = models.WrapAction(models.EmitProgressFor(
					&models.TimeoutAction{
						Action:              models.WrapAction(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"})))),
						DeprecatedTimeoutNs: 10 * int64(time.Second),
					},
					"start-message",
					"success-message",
					"failure-message",
				))
			})

			JustBeforeEach(func() {
				schedulingInfo, runInfo := desiredLRP.CreateComponents(fakeClock.Now())
				runInfo.DeprecatedStartTimeoutS = 15

				_, err := json.Marshal(desiredLRP.Routes)
				Expect(err).NotTo(HaveOccurred())
	})

	var newValidDesiredLRP = func(guid string) *models.DesiredLRP {
		myRouterJSON := json.RawMessage(`{"foo":"bar"}`)
		modTag := models.NewModificationTag("epoch", 0)
		desiredLRP := &models.DesiredLRP{
			ProcessGuid:             guid,
			Domain:                  "some-domain",
			RootFs:                  "some:rootfs",
			Instances:               1,
			EnvironmentVariables:    []*models.EnvironmentVariable{{Name: "FOO", Value: "bar"}},
			Setup:                   models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
			Action:                  models.WrapAction(&models.RunAction{Path: "ls", User: "******"}),
			DeprecatedStartTimeoutS: 15,
			Monitor: models.WrapAction(models.EmitProgressFor(
				models.Timeout(models.Try(models.Parallel(models.Serial(&models.RunAction{Path: "ls", User: "******"}))),
					10*time.Second,
				),
				"start-message",
				"success-message",
				"failure-message",
			)),
			DiskMb:      512,
			MemoryMb:    1024,
			CpuWeight:   42,
			Routes:      &models.Routes{"my-router": &myRouterJSON},
			LogSource:   "some-log-source",
			LogGuid:     "some-log-guid",
			MetricsGuid: "some-metrics-guid",
			Annotation:  "some-annotation",
			EgressRules: []*models.SecurityGroupRule{{
		BeforeEach(func() {
			newlyDesiredTask = &models.TaskDefinition{
				LogGuid:  "some-log-guid",
				MemoryMb: 128,
				DiskMb:   512,
				EnvironmentVariables: []*models.EnvironmentVariable{
					{Name: "foo", Value: "bar"},
					{Name: "VCAP_APPLICATION", Value: "{\"application_name\":\"my-app\"}"},
				},
				RootFs:                "http://docker-image.com",
				CompletionCallbackUrl: "http://api.cc.com/v1/tasks/complete",
				Action: models.WrapAction(models.Serial(
					&models.DownloadAction{
						From:     taskRequest.DropletUri,
						To:       ".",
						CacheKey: "",
						User:     "******",
					},
					&models.RunAction{},
				)),
			}

			buildpackBuilder.BuildTaskReturns(newlyDesiredTask, nil)
		})

		It("logs the incoming and outgoing request", func() {
			Eventually(logger.TestSink.Buffer).Should(gbytes.Say("serving"))
			Eventually(logger.TestSink.Buffer).Should(gbytes.Say("desiring-task"))
		})

		It("creates the task", func() {
Example #16
0
							"run": {
								"resource_limits": {},
								"path": "echo",
								"user": "******",
								"suppress_log_output": false
							}
						}
					]
			}`,
			models.WrapAction(models.Serial(
				&models.DownloadAction{
					From:     "web_location",
					To:       "local_location",
					CacheKey: "elephant",
					User:     "******",
				},
				&models.RunAction{
					Path:           "echo",
					User:           "******",
					ResourceLimits: &models.ResourceLimits{},
				},
			)),
		)

		Describe("Validate", func() {
			var serialAction *models.SerialAction

			Context("when the action has 'actions' as a slice of valid actions", func() {
				It("is valid", func() {
					serialAction = models.Serial(
						&models.UploadAction{
Example #17
0
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
	logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
	logger.Info("staging-request")

	var lifecycleData cc_messages.DockerStagingData
	err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	err = backend.validateRequest(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	compilerURL, err := backend.compilerDownloadURL()
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	cacheDockerImage := false
	for _, envVar := range request.Environment {
		if envVar.Name == "DIEGO_DOCKER_CACHE" && envVar.Value == "true" {
			cacheDockerImage = true
			break
		}
	}

	actions := []models.ActionInterface{}

	//Download builder
	actions = append(
		actions,
		models.EmitProgressFor(
			&models.DownloadAction{
				From:     compilerURL.String(),
				To:       path.Dir(DockerBuilderExecutablePath),
				CacheKey: "docker-lifecycle",
				User:     "******",
			},
			"",
			"",
			"Failed to set up docker environment",
		),
	)

	runActionArguments := []string{"-outputMetadataJSONFilename", DockerBuilderOutputPath, "-dockerRef", lifecycleData.DockerImageUrl}
	runAs := "vcap"
	if cacheDockerImage {
		runAs = "root"

		host, port, err := net.SplitHostPort(backend.config.DockerRegistryAddress)
		if err != nil {
			logger.Debug("invalid docker registry address", lager.Data{"address": backend.config.DockerRegistryAddress, "error": err.Error()})
			return &models.TaskDefinition{}, "", "", ErrInvalidDockerRegistryAddress
		}

		registryServices, err := getDockerRegistryServices(backend.config.ConsulCluster, backend.logger)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}
		registryRules := addDockerRegistryRules(request.EgressRules, registryServices)
		request.EgressRules = append(request.EgressRules, registryRules...)

		registryIPs := strings.Join(buildDockerRegistryAddresses(registryServices), ",")

		runActionArguments, err = addDockerCachingArguments(runActionArguments, registryIPs, backend.config.InsecureDockerRegistry, host, port, lifecycleData)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}
	}

	fileDescriptorLimit := uint64(request.FileDescriptors)

	// Run builder
	actions = append(
		actions,
		models.EmitProgressFor(
			&models.RunAction{
				Path: DockerBuilderExecutablePath,
				Args: runActionArguments,
				Env:  request.Environment,
				ResourceLimits: &models.ResourceLimits{
					Nofile: &fileDescriptorLimit,
				},
				User: runAs,
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		),
	)

	annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
		Lifecycle: DockerLifecycleName,
	})

	taskDefinition := &models.TaskDefinition{
		RootFs:                models.PreloadedRootFS(backend.config.DockerStagingStack),
		ResultFile:            DockerBuilderOutputPath,
		Privileged:            true,
		MemoryMb:              int32(request.MemoryMB),
		LogSource:             TaskLogSource,
		LogGuid:               request.LogGuid,
		EgressRules:           request.EgressRules,
		DiskMb:                int32(request.DiskMB),
		CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
		Annotation:            string(annotationJson),
		Action:                models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
	}
	logger.Debug("staging-task-request")

	return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
Example #18
0
			Expect(task.Failed).To(BeFalse())
		})

		Context("when the command exceeds its memory limit", func() {
			It("should fail the Task", func() {
				err := receptorClient.CreateTask(helpers.TaskCreateRequestWithMemoryAndDisk(
					guid,
					models.Serial(
						&models.RunAction{
							User: "******",
							Path: "curl",
							Args: []string{inigo_announcement_server.AnnounceURL("before-memory-overdose")},
						},
						&models.RunAction{
							User: "******",
							Path: "sh",
							Args: []string{"-c", "yes $(yes)"},
						},
						&models.RunAction{
							User: "******",
							Path: "curl",
							Args: []string{inigo_announcement_server.AnnounceURL("after-memory-overdose")},
						},
					),
					10,
					1024,
				))
				Expect(err).NotTo(HaveOccurred())

				Eventually(inigo_announcement_server.Announcements).Should(ContainElement("before-memory-overdose"))
Example #19
0
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
	logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
	logger.Info("staging-request")

	var lifecycleData cc_messages.DockerStagingData
	err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	err = backend.validateRequest(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	compilerURL, err := backend.compilerDownloadURL()
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	cachedDependencies := []*models.CachedDependency{
		&models.CachedDependency{
			From:     compilerURL.String(),
			To:       path.Dir(DockerBuilderExecutablePath),
			CacheKey: "docker-lifecycle",
		},
	}

	runActionArguments := []string{
		"-outputMetadataJSONFilename", DockerBuilderOutputPath,
		"-dockerRef", lifecycleData.DockerImageUrl,
	}

	if len(backend.config.InsecureDockerRegistries) > 0 {
		insecureDockerRegistries := strings.Join(backend.config.InsecureDockerRegistries, ",")
		runActionArguments = append(runActionArguments, "-insecureDockerRegistries", insecureDockerRegistries)
	}

	fileDescriptorLimit := uint64(request.FileDescriptors)
	runAs := "vcap"

	actions := []models.ActionInterface{}

	if cacheDockerImage(request.Environment) {
		runAs = "root"

		additionalEgressRules, additionalArgs, err := cachingEgressRulesAndArgs(
			logger,
			backend.config.DockerRegistryAddress,
			backend.config.ConsulCluster,
			lifecycleData,
		)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}

		runActionArguments = append(runActionArguments, additionalArgs...)
		request.EgressRules = append(request.EgressRules, additionalEgressRules...)

		actions = append(
			actions,
			models.EmitProgressFor(
				&models.RunAction{
					Path: MountCgroupsPath,
					ResourceLimits: &models.ResourceLimits{
						Nofile: &fileDescriptorLimit,
					},
					User: runAs,
				},
				"Preparing docker daemon...",
				"",
				"Failed to set up docker environment",
			),
		)
	}

	actions = append(
		actions,
		models.EmitProgressFor(
			&models.RunAction{
				Path: DockerBuilderExecutablePath,
				Args: runActionArguments,
				Env:  request.Environment,
				ResourceLimits: &models.ResourceLimits{
					Nofile: &fileDescriptorLimit,
				},
				User: runAs,
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		),
	)

	annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
		Lifecycle:          DockerLifecycleName,
		CompletionCallback: request.CompletionCallback,
	})

	taskDefinition := &models.TaskDefinition{
		RootFs:                        models.PreloadedRootFS(backend.config.DockerStagingStack),
		ResultFile:                    DockerBuilderOutputPath,
		Privileged:                    true,
		MemoryMb:                      int32(request.MemoryMB),
		LogSource:                     TaskLogSource,
		LogGuid:                       request.LogGuid,
		EgressRules:                   request.EgressRules,
		DiskMb:                        int32(request.DiskMB),
		CompletionCallbackUrl:         backend.config.CallbackURL(stagingGuid),
		Annotation:                    string(annotationJson),
		Action:                        models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
		CachedDependencies:            cachedDependencies,
		LegacyDownloadUser:            "******",
		TrustedSystemCertificatesPath: TrustedSystemCertificatesPath,
	}
	logger.Debug("staging-task-request")

	return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
				Expect(desiredLRP.StartTimeout).To(BeEquivalentTo(123456))

				Expect(desiredLRP.LogGuid).To(Equal("the-log-id"))
				Expect(desiredLRP.LogSource).To(Equal("CELL"))

				Expect(desiredLRP.EnvironmentVariables).To(ConsistOf(&models.EnvironmentVariable{"LANG", recipebuilder.DefaultLANG}))

				Expect(desiredLRP.MetricsGuid).To(Equal("the-log-id"))

				expectedSetup := models.Serial(
					&models.DownloadAction{
						From:     "http://file-server.com/v1/static/some-lifecycle.tgz",
						To:       "/tmp/lifecycle",
						CacheKey: "buildpack-some-stack-lifecycle",
						User:     "******",
					},
					&models.DownloadAction{
						From:     "http://the-droplet.uri.com",
						To:       ".",
						CacheKey: "droplets-the-app-guid-the-app-version",
						User:     "******",
					},
				)
				Expect(desiredLRP.Setup.GetValue()).To(Equal(expectedSetup))

				parallelRunAction := desiredLRP.Action.CodependentAction
				Expect(parallelRunAction.Actions).To(HaveLen(1))

				runAction := parallelRunAction.Actions[0].RunAction

				Expect(desiredLRP.Monitor.GetValue()).To(Equal(models.Timeout(
					&models.ParallelAction{