Пример #1
0
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
	logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
	logger.Info("staging-request")

	var lifecycleData cc_messages.DockerStagingData
	err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	err = backend.validateRequest(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	compilerURL, err := backend.compilerDownloadURL()
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	cachedDependencies := []*models.CachedDependency{
		&models.CachedDependency{
			From:     compilerURL.String(),
			To:       path.Dir(DockerBuilderExecutablePath),
			CacheKey: "docker-lifecycle",
		},
	}

	runActionArguments := []string{
		"-outputMetadataJSONFilename", DockerBuilderOutputPath,
		"-dockerRef", lifecycleData.DockerImageUrl,
	}

	if len(backend.config.InsecureDockerRegistries) > 0 {
		insecureDockerRegistries := strings.Join(backend.config.InsecureDockerRegistries, ",")
		runActionArguments = append(runActionArguments, "-insecureDockerRegistries", insecureDockerRegistries)
	}

	fileDescriptorLimit := uint64(request.FileDescriptors)
	runAs := "vcap"

	actions := []models.ActionInterface{}

	if cacheDockerImage(request.Environment) {
		runAs = "root"

		additionalEgressRules, additionalArgs, err := cachingEgressRulesAndArgs(
			logger,
			backend.config.DockerRegistryAddress,
			backend.config.ConsulCluster,
			lifecycleData,
		)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}

		runActionArguments = append(runActionArguments, additionalArgs...)
		request.EgressRules = append(request.EgressRules, additionalEgressRules...)

		actions = append(
			actions,
			models.EmitProgressFor(
				&models.RunAction{
					Path: MountCgroupsPath,
					ResourceLimits: &models.ResourceLimits{
						Nofile: &fileDescriptorLimit,
					},
					User: runAs,
				},
				"Preparing docker daemon...",
				"",
				"Failed to set up docker environment",
			),
		)
	}

	actions = append(
		actions,
		models.EmitProgressFor(
			&models.RunAction{
				Path: DockerBuilderExecutablePath,
				Args: runActionArguments,
				Env:  request.Environment,
				ResourceLimits: &models.ResourceLimits{
					Nofile: &fileDescriptorLimit,
				},
				User: runAs,
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		),
	)

	annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
		Lifecycle:          DockerLifecycleName,
		CompletionCallback: request.CompletionCallback,
	})

	taskDefinition := &models.TaskDefinition{
		RootFs:                        models.PreloadedRootFS(backend.config.DockerStagingStack),
		ResultFile:                    DockerBuilderOutputPath,
		Privileged:                    true,
		MemoryMb:                      int32(request.MemoryMB),
		LogSource:                     TaskLogSource,
		LogGuid:                       request.LogGuid,
		EgressRules:                   request.EgressRules,
		DiskMb:                        int32(request.DiskMB),
		CompletionCallbackUrl:         backend.config.CallbackURL(stagingGuid),
		Annotation:                    string(annotationJson),
		Action:                        models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
		CachedDependencies:            cachedDependencies,
		LegacyDownloadUser:            "******",
		TrustedSystemCertificatesPath: TrustedSystemCertificatesPath,
	}
	logger.Debug("staging-task-request")

	return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
Пример #2
0
func (backend *dockerBackend) BuildRecipe(stagingGuid string, request cc_messages.StagingRequestFromCC) (*models.TaskDefinition, string, string, error) {
	logger := backend.logger.Session("build-recipe", lager.Data{"app-id": request.AppId, "staging-guid": stagingGuid})
	logger.Info("staging-request")

	var lifecycleData cc_messages.DockerStagingData
	err := json.Unmarshal(*request.LifecycleData, &lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	err = backend.validateRequest(request, lifecycleData)
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	compilerURL, err := backend.compilerDownloadURL()
	if err != nil {
		return &models.TaskDefinition{}, "", "", err
	}

	cacheDockerImage := false
	for _, envVar := range request.Environment {
		if envVar.Name == "DIEGO_DOCKER_CACHE" && envVar.Value == "true" {
			cacheDockerImage = true
			break
		}
	}

	actions := []models.ActionInterface{}

	//Download builder
	actions = append(
		actions,
		models.EmitProgressFor(
			&models.DownloadAction{
				From:     compilerURL.String(),
				To:       path.Dir(DockerBuilderExecutablePath),
				CacheKey: "docker-lifecycle",
				User:     "******",
			},
			"",
			"",
			"Failed to set up docker environment",
		),
	)

	runActionArguments := []string{"-outputMetadataJSONFilename", DockerBuilderOutputPath, "-dockerRef", lifecycleData.DockerImageUrl}
	runAs := "vcap"
	if cacheDockerImage {
		runAs = "root"

		host, port, err := net.SplitHostPort(backend.config.DockerRegistryAddress)
		if err != nil {
			logger.Debug("invalid docker registry address", lager.Data{"address": backend.config.DockerRegistryAddress, "error": err.Error()})
			return &models.TaskDefinition{}, "", "", ErrInvalidDockerRegistryAddress
		}

		registryServices, err := getDockerRegistryServices(backend.config.ConsulCluster, backend.logger)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}
		registryRules := addDockerRegistryRules(request.EgressRules, registryServices)
		request.EgressRules = append(request.EgressRules, registryRules...)

		registryIPs := strings.Join(buildDockerRegistryAddresses(registryServices), ",")

		runActionArguments, err = addDockerCachingArguments(runActionArguments, registryIPs, backend.config.InsecureDockerRegistry, host, port, lifecycleData)
		if err != nil {
			return &models.TaskDefinition{}, "", "", err
		}
	}

	fileDescriptorLimit := uint64(request.FileDescriptors)

	// Run builder
	actions = append(
		actions,
		models.EmitProgressFor(
			&models.RunAction{
				Path: DockerBuilderExecutablePath,
				Args: runActionArguments,
				Env:  request.Environment,
				ResourceLimits: &models.ResourceLimits{
					Nofile: &fileDescriptorLimit,
				},
				User: runAs,
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		),
	)

	annotationJson, _ := json.Marshal(cc_messages.StagingTaskAnnotation{
		Lifecycle: DockerLifecycleName,
	})

	taskDefinition := &models.TaskDefinition{
		RootFs:                models.PreloadedRootFS(backend.config.DockerStagingStack),
		ResultFile:            DockerBuilderOutputPath,
		Privileged:            true,
		MemoryMb:              int32(request.MemoryMB),
		LogSource:             TaskLogSource,
		LogGuid:               request.LogGuid,
		EgressRules:           request.EgressRules,
		DiskMb:                int32(request.DiskMB),
		CompletionCallbackUrl: backend.config.CallbackURL(stagingGuid),
		Annotation:            string(annotationJson),
		Action:                models.WrapAction(models.Timeout(models.Serial(actions...), dockerTimeout(request, backend.logger))),
	}
	logger.Debug("staging-task-request")

	return taskDefinition, stagingGuid, backend.config.TaskDomain, nil
}
Пример #3
0
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
)

var _ = Describe("DockerBackend", func() {
	var (
		stagingRequest        cc_messages.StagingRequestFromCC
		downloadBuilderAction models.ActionInterface
		runAction             models.ActionInterface
		config                backend.Config
		logger                lager.Logger
		docker                backend.Backend

		stagingGuid       string
		appId             string
		dockerImageUrl    string
		dockerLoginServer string
		dockerUser        string
		dockerPassword    string
		dockerEmail       string
		fileDescriptors   int
		memoryMb          int32
		diskMb            int32
		timeout           int
		egressRules       []*models.SecurityGroupRule
	)

	BeforeEach(func() {
		appId = "bunny"
		dockerImageUrl = "busybox"
		dockerLoginServer = ""
)

var _ = Describe("TraditionalBackend", func() {
	var (
		traditional                    backend.Backend
		stagingRequest                 cc_messages.StagingRequestFromCC
		config                         backend.Config
		buildpackOrder                 string
		timeout                        int
		stack                          string
		memoryMb                       int32
		diskMb                         int32
		fileDescriptors                int
		buildArtifactsCacheDownloadUri string
		appId                          string
		stagingGuid                    string
		buildpacks                     []cc_messages.Buildpack
		appBitsDownloadUri             string
		downloadBuilderAction          models.ActionInterface
		downloadAppAction              models.ActionInterface
		downloadFirstBuildpackAction   models.ActionInterface
		downloadSecondBuildpackAction  models.ActionInterface
		downloadBuildArtifactsAction   models.ActionInterface
		runAction                      models.ActionInterface
		uploadDropletAction            models.ActionInterface
		uploadBuildArtifactsAction     models.ActionInterface
		egressRules                    []*models.SecurityGroupRule
		environment                    []*models.EnvironmentVariable
	)

	BeforeEach(func() {
		stagerURL := "http://the-stager.example.com"
Пример #5
0
			DockerPassword:    password,
			DockerEmail:       email,
		})
		Expect(err).NotTo(HaveOccurred())
		lifecycleData := json.RawMessage(rawJsonBytes)

		Expect(err).NotTo(HaveOccurred())

		stagingRequest := cc_messages.StagingRequestFromCC{
			AppId:           "bunny",
			FileDescriptors: 512,
			MemoryMB:        512,
			DiskMB:          512,
			Timeout:         512,
			LifecycleData:   &lifecycleData,
			EgressRules: []*models.SecurityGroupRule{
				{
					Protocol:     "TCP",
					Destinations: []string{"0.0.0.0/0"},
					PortRange:    &models.PortRange{Start: 80, End: 443},
				},
			},
		}

		if dockerImageCachingEnabled {
			stagingRequest.Environment = append(stagingRequest.Environment, &models.EnvironmentVariable{
				Name:  "DIEGO_DOCKER_CACHE",
				Value: "true",
			})
		}
Пример #6
0
			LifecycleData:   &lifecycleData,
		}
	}

	BeforeEach(func() {
		loginServer = ""
		user = ""
		password = ""
		email = ""
	})

	Context("when docker registry is running", func() {
		var (
			downloadBuilderAction  models.ActionInterface
			docker                 backend.Backend
			expectedRunAction      models.ActionInterface
			expectedEgressRules    []*models.SecurityGroupRule
			insecureDockerRegistry bool
			stagingRequest         cc_messages.StagingRequestFromCC
		)

		setupEgressRules := func(ips []string) []*models.SecurityGroupRule {
			rules := []*models.SecurityGroupRule{}
			for _, ip := range ips {
				rules = append(rules, &models.SecurityGroupRule{
					Protocol:     models.TCPProtocol,
					Destinations: []string{ip},
					Ports:        []uint32{dockerRegistryPort},
				})
			}
			return rules
		}