Exemple #1
0
func assertFixture(t *testing.T, name string) {
	orig := manifest.ManifestRandomPorts
	manifest.ManifestRandomPorts = false
	defer func() { manifest.ManifestRandomPorts = orig }()

	app := models.App{
		Name: "httpd",
		Tags: map[string]string{
			"Name":   "httpd",
			"Type":   "app",
			"System": "convox",
			"Rack":   "convox-test",
		},
	}

	data, err := ioutil.ReadFile(fmt.Sprintf("fixtures/%s.yml", name))
	require.Nil(t, err)

	manifest, err := manifest.Load(data)
	require.Nil(t, err)

	formation, err := app.Formation(*manifest)
	require.Nil(t, err)

	pretty, err := models.PrettyJSON(formation)
	require.Nil(t, err)

	data, err = ioutil.ReadFile(fmt.Sprintf("fixtures/%s.json", name))
	require.Nil(t, err)

	diff1 := strings.Split(strings.TrimSpace(string(data)), "\n")
	diff2 := strings.Split(strings.TrimSpace(pretty), "\n")

	diff := difflib.Diff(diff1, diff2)
	diffs := []string{}

	// bigger than max
	prev := 1000000

	for l, d := range diff {
		switch d.Delta {
		case difflib.LeftOnly:
			if (l - prev) > 1 {
				diffs = append(diffs, "")
			}
			diffs = append(diffs, fmt.Sprintf("%04d - %s", l, d.Payload))
			prev = l
		case difflib.RightOnly:
			if (l - prev) > 1 {
				diffs = append(diffs, "")
			}
			diffs = append(diffs, fmt.Sprintf("%04d + %s", l, d.Payload))
			prev = l
		}
	}

	if len(diffs) > 0 {
		t.Errorf("Unexpected results for %s:\n%s", name, strings.Join(diffs, "\n"))
	}
}
Exemple #2
0
func TestLoadGarbage(t *testing.T) {
	m, err := manifest.Load([]byte("\t\003//783bfkl1f"))

	if assert.Nil(t, m) && assert.NotNil(t, err) {
		assert.Equal(t, err.Error(), "could not parse manifest: yaml: control characters are not allowed")
	}
}
Exemple #3
0
func (r *Release) Formation() (string, error) {
	app, err := GetApp(r.App)
	if err != nil {
		return "", err
	}

	manifest, err := manifest.Load([]byte(r.Manifest))
	if err != nil {
		return "", err
	}

	// Bound apps do not use the StackName as ELB name.
	if !app.IsBound() {
		// try to figure out which process to map to the main load balancer
		primary, err := primaryProcess(app.StackName())

		if err != nil {
			return "", err
		}

		// if we dont have a primary default to a process named web
		_, ok := manifest.Services["web"]
		if primary == "" && ok {
			primary = "web"
		}

		// if we still dont have a primary try the first process with external ports
		if primary == "" && manifest.HasExternalPorts() {
			for _, entry := range manifest.Services {
				if len(entry.ExternalPorts()) > 0 {
					primary = entry.Name
					break
				}
			}
		}

		for _, entry := range manifest.Services {
			if entry.Name == primary {
				//TODO not sure this indirection is required
				dup := manifest.Services[entry.Name]
				dup.Primary = true
				manifest.Services[entry.Name] = dup
			}
		}
	}

	// set the image
	for i, entry := range manifest.Services {
		s := manifest.Services[i]
		s.Image = entry.RegistryImage(app.Name, r.Build, app.Outputs)
		manifest.Services[i] = s
	}

	manifest, err = r.resolveLinks(*app, manifest)
	if err != nil {
		return "", err
	}

	return app.Formation(*manifest)
}
Exemple #4
0
// FormationGet gets a Formation
func (p *AWSProvider) FormationGet(app, process string) (*structs.ProcessFormation, error) {
	a, err := p.AppGet(app)
	if err != nil {
		return nil, err
	}
	if a.Release == "" {
		return nil, fmt.Errorf("no release for app: %s", app)
	}

	release, err := p.ReleaseGet(a.Name, a.Release)
	if err != nil {
		return nil, err
	}

	manifest, err := manifest.Load([]byte(release.Manifest))
	if err != nil {
		return nil, fmt.Errorf("could not parse manifest for release: %s", release.Id)
	}

	if _, ok := manifest.Services[process]; !ok {
		return nil, fmt.Errorf("no such process: %s", process)
	}

	return processFormation(a, manifest.Services[process])
}
Exemple #5
0
func build(dir string) error {
	dcy := filepath.Join(dir, flagConfig)

	if _, err := os.Stat(dcy); os.IsNotExist(err) {
		return fmt.Errorf("no such file: %s", flagConfig)
	}

	data, err := ioutil.ReadFile(dcy)
	if err != nil {
		return err
	}

	m, err := manifest.Load(data)
	if err != nil {
		return err
	}

	errs := m.Validate()
	if len(errs) > 0 {
		return errs[0]
	}

	s := make(chan string)

	go func() {
		for l := range s {
			log(l)
		}
	}()

	defer close(s)

	err = m.Build(dir, flagApp, s, manifest.BuildOptions{
		Cache: flagCache == "true",
	})
	if err != nil {
		return err
	}

	if err := m.Push(flagPush, flagApp, flagId, s); err != nil {
		return err
	}

	return nil
}
Exemple #6
0
// deleteImages generates a list of fully qualified URLs for images for every process type
// in the build manifest then deletes them.
// Image URLs that point to ECR, e.g. 826133048.dkr.ecr.us-east-1.amazonaws.com/myapp-zridvyqapp:web.BSUSBFCUCSA,
// are deleted with the ECR BatchDeleteImage API.
// Image URLs that point to the convox-hosted registry, e.g. convox-826133048.us-east-1.elb.amazonaws.com:5000/myapp-web:BSUSBFCUCSA,
// are not yet supported and return an error.
func (p *AWSProvider) deleteImages(a *structs.App, b *structs.Build) error {

	m, err := manifest.Load([]byte(b.Manifest))
	if err != nil {
		return err
	}

	// failed builds could have an empty manifest
	if len(m.Services) == 0 {
		return nil
	}

	urls := []string{}

	for name := range m.Services {
		urls = append(urls, p.registryTag(a, name, b.Id))
	}

	imageIds := []*ecr.ImageIdentifier{}
	registryId := ""
	repositoryName := ""

	for _, url := range urls {
		if match := regexpECRImage.FindStringSubmatch(url); match != nil {
			registryId = match[1]
			repositoryName = match[3]

			imageIds = append(imageIds, &ecr.ImageIdentifier{
				ImageTag: aws.String(match[4]),
			})
		} else {
			return errors.New("URL not valid ECR")
		}
	}

	_, err = p.ecr().BatchDeleteImage(&ecr.BatchDeleteImageInput{
		ImageIds:       imageIds,
		RegistryId:     aws.String(registryId),
		RepositoryName: aws.String(repositoryName),
	})

	return err
}
Exemple #7
0
func main() {
	if len(os.Args) < 2 {
		die(fmt.Errorf("usage: fixture <docker-compose.yml>"))
	}

	os.Setenv("REGION", "test")

	data, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		die(err)
	}

	app := models.App{
		Name: "httpd",
		Tags: map[string]string{
			"Name":   "httpd",
			"Type":   "app",
			"System": "convox",
			"Rack":   "convox-test",
		},
	}

	m, err := manifest.Load(data)
	if err != nil {
		die(err)
	}

	f, err := app.Formation(*m)
	if err != nil {
		die(err)
	}

	pretty, err := models.PrettyJSON(f)
	if err != nil {
		die(err)
	}

	fmt.Println(pretty)
}
Exemple #8
0
// FormationList lists the Formation
func (p *AWSProvider) FormationList(app string) (structs.Formation, error) {
	log := Logger.At("FormationList").Start()

	a, err := p.AppGet(app)
	if err != nil {
		log.Error(err)
		return nil, err
	}
	if a.Release == "" {
		return structs.Formation{}, nil
	}

	release, err := p.ReleaseGet(a.Name, a.Release)
	if err != nil {
		log.Error(err)
		return nil, err
	}

	manifest, err := manifest.Load([]byte(release.Manifest))
	if err != nil {
		return nil, fmt.Errorf("could not parse manifest for release: %s", release.Id)
	}

	formation := structs.Formation{}

	for _, s := range manifest.Services {
		pf, err := processFormation(a, s)
		if err != nil {
			return nil, err
		}

		formation = append(formation, *pf)
	}

	log.Success()

	return formation, nil
}
Exemple #9
0
func (r *Release) Promote() error {
	app, err := GetApp(r.App)
	if err != nil {
		return err
	}

	if !app.IsBound() {
		return fmt.Errorf("unbound apps are no longer supported for promotion")
	}

	formation, err := r.Formation()
	if err != nil {
		return err
	}

	// If release formation was saved in S3, get that instead
	f, err := s3Get(app.Outputs["Settings"], fmt.Sprintf("templates/%s", r.Id))
	if err != nil && awserrCode(err) != "NoSuchKey" {
		return err
	}
	if err == nil {
		formation = string(f)
	}

	fmt.Printf("ns=kernel at=release.promote at=s3Get found=%t\n", err == nil)

	existing, err := formationParameters(formation)
	if err != nil {
		return err
	}

	app.Parameters["Environment"] = r.EnvironmentUrl()
	app.Parameters["Kernel"] = CustomTopic
	app.Parameters["Release"] = r.Id
	app.Parameters["Version"] = os.Getenv("RELEASE")
	app.Parameters["VPCCIDR"] = os.Getenv("VPCCIDR")

	if os.Getenv("ENCRYPTION_KEY") != "" {
		app.Parameters["Key"] = os.Getenv("ENCRYPTION_KEY")
	}

	// SubnetsPrivate is a List<AWS::EC2::Subnet::Id> and can not be empty
	// So reuse SUBNETS if SUBNETS_PRIVATE is not set
	subnetsPrivate := os.Getenv("SUBNETS_PRIVATE")
	if subnetsPrivate == "" {
		subnetsPrivate = os.Getenv("SUBNETS")
	}

	app.Parameters["SubnetsPrivate"] = subnetsPrivate

	m, err := manifest.Load([]byte(r.Manifest))
	if err != nil {
		return err
	}

	// set default values for memory and cpu

	for _, entry := range m.Services {
		scale := []string{"1", "0", "256"}

		if entry.Cpu > 0 {
			scale[1] = fmt.Sprintf("%d", entry.Cpu)
		}

		if entry.Memory > 0 {
			scale[2] = fmt.Sprintf("%d", int64(math.Ceil(float64(entry.Memory)/(1024*1024))))
		}

		switch {
		case app.Parameters[entry.ParamName("Formation")] != "":
			scale = strings.Split(app.Parameters[entry.ParamName("Formation")], ",")
			if len(scale) != 3 {
				return fmt.Errorf("%s not in Count,Cpu,Memory format", entry.ParamName("Formation"))
			}
		case app.Parameters[entry.ParamName("DesiredCount")] != "":
			if v, ok := app.Parameters[entry.ParamName("DesiredCount")]; ok {
				scale[0] = v
			}
			if v, ok := app.Parameters[entry.ParamName("Cpu")]; ok {
				scale[1] = v
			}
			if v, ok := app.Parameters[entry.ParamName("Memory")]; ok {
				scale[2] = v
			}
		}

		app.Parameters[entry.ParamName("Formation")] = strings.Join(scale, ",")

		// backwards compatibility for rollbacks
		app.Parameters[entry.ParamName("DesiredCount")] = scale[0]
		app.Parameters[entry.ParamName("Cpu")] = scale[1]
		app.Parameters[entry.ParamName("Memory")] = scale[2]

		for _, mapping := range entry.Ports {
			certParam := fmt.Sprintf("%sPort%dCertificate", UpperName(entry.Name), mapping.Balancer)
			protoParam := fmt.Sprintf("%sPort%dProtocol", UpperName(entry.Name), mapping.Balancer)
			proxyParam := fmt.Sprintf("%sPort%dProxy", UpperName(entry.Name), mapping.Balancer)
			secureParam := fmt.Sprintf("%sPort%dSecure", UpperName(entry.Name), mapping.Balancer)

			proto := entry.Labels[fmt.Sprintf("convox.port.%d.protocol", mapping.Balancer)]

			// if the proto param is set to a non-default value and doesnt match the label, error
			if ap, ok := app.Parameters[protoParam]; ok {
				if ap != "tcp" && ap != proto {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.protocol label instead", protoParam, mapping.Balancer)
				}
			}

			// if the proxy param is set and doesnt match the label, error
			if ap, ok := app.Parameters[proxyParam]; ok {
				if ap == "Yes" && entry.Labels[fmt.Sprintf("convox.port.%d.proxy", mapping.Balancer)] != "true" {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.proxy label instead", proxyParam, mapping.Balancer)
				}
			}

			// if the secure param is set and doesnt match the label, error
			if ap, ok := app.Parameters[secureParam]; ok {
				if ap == "Yes" && entry.Labels[fmt.Sprintf("convox.port.%d.secure", mapping.Balancer)] != "true" {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.secure label instead", secureParam, mapping.Balancer)
				}
			}

			switch proto {
			case "https", "tls":
				if app.Parameters[certParam] == "" {
					// if rack already has a self-signed cert, reuse it
					certs, err := IAM().ListServerCertificates(&iam.ListServerCertificatesInput{})
					if err != nil {
						return err
					}

					for _, cert := range certs.ServerCertificateMetadataList {
						if strings.Contains(*cert.Arn, fmt.Sprintf("cert-%s", os.Getenv("RACK"))) {
							app.Parameters[certParam] = *cert.Arn
							break
						}
					}

					// if not, generate and upload a self-signed cert
					if app.Parameters[certParam] == "" {
						name := fmt.Sprintf("cert-%s-%d-%05d", os.Getenv("RACK"), time.Now().Unix(), rand.Intn(100000))

						body, key, err := generateSelfSignedCertificate("*.*.elb.amazonaws.com")
						if err != nil {
							return err
						}

						input := &iam.UploadServerCertificateInput{
							CertificateBody:       aws.String(string(body)),
							PrivateKey:            aws.String(string(key)),
							ServerCertificateName: aws.String(name),
						}

						res, err := IAM().UploadServerCertificate(input)
						if err != nil {
							return err
						}

						app.Parameters[certParam] = *res.ServerCertificateMetadata.Arn

						// We want to make sure the new cert has propagated throughout AWS before moving on
						sleep := 5
						for {
							time.Sleep(time.Duration(sleep) * time.Second)

							_, err := IAM().GetServerCertificate(&iam.GetServerCertificateInput{
								ServerCertificateName: &name,
							})
							if err != nil {
								if ae, ok := err.(awserr.Error); ok {
									if ae.Code() == "NoSuchEntity" {
										if sleep >= 40 {
											return fmt.Errorf("certificate `%s` was not found", name)
										}

										sleep *= 2
										continue
									}

								}
								return err
							}
							break
						}
					}
				}
			}
		}
	}

	params := []*cloudformation.Parameter{}

	for key, value := range app.Parameters {
		if _, ok := existing[key]; ok {
			params = append(params, &cloudformation.Parameter{ParameterKey: aws.String(key), ParameterValue: aws.String(value)})
		}
	}

	err = S3Put(app.Outputs["Settings"], fmt.Sprintf("templates/%s", r.Id), []byte(formation), false)
	if err != nil {
		return err
	}

	// loop until we can find the template
	if err := waitForTemplate(app.Outputs["Settings"], r.Id); err != nil {
		return fmt.Errorf("error waiting for template: %s", err)
	}

	url := fmt.Sprintf("https://s3.amazonaws.com/%s/templates/%s", app.Outputs["Settings"], r.Id)

	req := &cloudformation.UpdateStackInput{
		Capabilities: []*string{aws.String("CAPABILITY_IAM")},
		StackName:    aws.String(app.StackName()),
		TemplateURL:  aws.String(url),
		Parameters:   params,
	}

	_, err = UpdateStack(req)

	NotifySuccess("release:promote", map[string]string{
		"app": r.App,
		"id":  r.Id,
	})

	return err
}
Exemple #10
0
func (r *Release) Promote() error {
	app, err := GetApp(r.App)
	if err != nil {
		return err
	}

	if !app.IsBound() {
		return fmt.Errorf("unbound apps are no longer supported for promotion")
	}

	formation, err := r.Formation()
	if err != nil {
		return err
	}

	// If release formation was saved in S3, get that instead
	f, err := s3Get(app.Outputs["Settings"], fmt.Sprintf("templates/%s", r.Id))
	if err != nil && awserrCode(err) != "NoSuchKey" {
		return err
	}
	if err == nil {
		formation = string(f)
	}

	fmt.Printf("ns=kernel at=release.promote at=s3Get found=%t\n", err == nil)

	existing, err := formationParameters(formation)
	if err != nil {
		return err
	}

	oldVersion := app.Parameters["Version"]

	app.Parameters["Environment"] = r.EnvironmentUrl()
	app.Parameters["Kernel"] = CustomTopic
	app.Parameters["Release"] = r.Id
	app.Parameters["Version"] = os.Getenv("RELEASE")
	app.Parameters["VPCCIDR"] = os.Getenv("VPCCIDR")

	if os.Getenv("ENCRYPTION_KEY") != "" {
		app.Parameters["Key"] = os.Getenv("ENCRYPTION_KEY")
	}

	// SubnetsPrivate is a List<AWS::EC2::Subnet::Id> and can not be empty
	// So reuse SUBNETS if SUBNETS_PRIVATE is not set
	subnetsPrivate := os.Getenv("SUBNETS_PRIVATE")
	if subnetsPrivate == "" {
		subnetsPrivate = os.Getenv("SUBNETS")
	}

	app.Parameters["SubnetsPrivate"] = subnetsPrivate

	m, err := manifest.Load([]byte(r.Manifest))
	if err != nil {
		return err
	}

	for _, entry := range m.Services {
		// set all of WebCount=1, WebCpu=0, WebMemory=256 and WebFormation=1,0,256 style parameters
		// so new deploys and rollbacks have the expected parameters
		if vals, ok := app.Parameters[fmt.Sprintf("%sFormation", UpperName(entry.Name))]; ok {
			parts := strings.SplitN(vals, ",", 3)
			if len(parts) != 3 {
				return fmt.Errorf("%s formation settings not in Count,Cpu,Memory format", entry.Name)
			}

			_, err = strconv.Atoi(parts[0])
			if err != nil {
				return fmt.Errorf("%s %s not numeric", entry.Name, "count")
			}

			_, err = strconv.Atoi(parts[1])
			if err != nil {
				return fmt.Errorf("%s %s not numeric", entry.Name, "CPU")
			}

			_, err = strconv.Atoi(parts[2])
			if err != nil {
				return fmt.Errorf("%s %s not numeric", entry.Name, "memory")
			}

			app.Parameters[fmt.Sprintf("%sDesiredCount", UpperName(entry.Name))] = parts[0]
			app.Parameters[fmt.Sprintf("%sCpu", UpperName(entry.Name))] = parts[1]
			app.Parameters[fmt.Sprintf("%sMemory", UpperName(entry.Name))] = parts[2]
		} else {
			parts := []string{"1", "0", "256"}

			if v := app.Parameters[fmt.Sprintf("%sDesiredCount", UpperName(entry.Name))]; v != "" {
				parts[0] = v
			}

			if v := app.Parameters[fmt.Sprintf("%sCpu", UpperName(entry.Name))]; v != "" {
				parts[1] = v
			}

			if v := app.Parameters[fmt.Sprintf("%sMemory", UpperName(entry.Name))]; v != "" {
				parts[2] = v
			}

			app.Parameters[fmt.Sprintf("%sFormation", UpperName(entry.Name))] = strings.Join(parts, ",")
		}

		for _, mapping := range entry.Ports {
			certParam := fmt.Sprintf("%sPort%dCertificate", UpperName(entry.Name), mapping.Balancer)
			protoParam := fmt.Sprintf("%sPort%dProtocol", UpperName(entry.Name), mapping.Balancer)
			proxyParam := fmt.Sprintf("%sPort%dProxy", UpperName(entry.Name), mapping.Balancer)
			secureParam := fmt.Sprintf("%sPort%dSecure", UpperName(entry.Name), mapping.Balancer)

			proto := entry.Labels[fmt.Sprintf("convox.port.%d.protocol", mapping.Balancer)]

			// if the proto param is set to a non-default value and doesnt match the label, error
			if ap, ok := app.Parameters[protoParam]; ok {
				if ap != "tcp" && ap != proto {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.protocol label instead", protoParam, mapping.Balancer)
				}
			}

			// if the proxy param is set and doesnt match the label, error
			if ap, ok := app.Parameters[proxyParam]; ok {
				if ap == "Yes" && entry.Labels[fmt.Sprintf("convox.port.%d.proxy", mapping.Balancer)] != "true" {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.proxy label instead", proxyParam, mapping.Balancer)
				}
			}

			// if the secure param is set and doesnt match the label, error
			if ap, ok := app.Parameters[secureParam]; ok {
				if ap == "Yes" && entry.Labels[fmt.Sprintf("convox.port.%d.secure", mapping.Balancer)] != "true" {
					return fmt.Errorf("%s parameter has been deprecated. Please set the convox.port.%d.secure label instead", secureParam, mapping.Balancer)
				}
			}

			switch proto {
			case "https", "tls":
				if app.Parameters[certParam] == "" {
					name := fmt.Sprintf("cert-%s-%d-%05d", os.Getenv("RACK"), time.Now().Unix(), rand.Intn(100000))

					body, key, err := generateSelfSignedCertificate("*.*.elb.amazonaws.com")
					if err != nil {
						return err
					}

					input := &iam.UploadServerCertificateInput{
						CertificateBody:       aws.String(string(body)),
						PrivateKey:            aws.String(string(key)),
						ServerCertificateName: aws.String(name),
					}

					// upload certificate
					res, err := IAM().UploadServerCertificate(input)
					if err != nil {
						return err
					}

					app.Parameters[certParam] = *res.ServerCertificateMetadata.Arn
				}
			}
		}
	}

	// randomize the instance ports for older apps so we can upgrade smoothly
	if oldVersion < "20160818013241" {
		for key := range app.Parameters {
			if strings.HasSuffix(key, "Host") {
				app.Parameters[key] = strconv.Itoa(rand.Intn(50000) + 10000)
			}
		}
	}

	params := []*cloudformation.Parameter{}

	for key, value := range app.Parameters {
		if _, ok := existing[key]; ok {
			params = append(params, &cloudformation.Parameter{ParameterKey: aws.String(key), ParameterValue: aws.String(value)})
		}
	}

	err = S3Put(app.Outputs["Settings"], fmt.Sprintf("templates/%s", r.Id), []byte(formation), false)
	if err != nil {
		return err
	}

	// loop until we can find the template
	if err := waitForTemplate(app.Outputs["Settings"], r.Id); err != nil {
		return fmt.Errorf("error waiting for template: %s", err)
	}

	url := fmt.Sprintf("https://s3.amazonaws.com/%s/templates/%s", app.Outputs["Settings"], r.Id)

	req := &cloudformation.UpdateStackInput{
		Capabilities: []*string{aws.String("CAPABILITY_IAM")},
		StackName:    aws.String(app.StackName()),
		TemplateURL:  aws.String(url),
		Parameters:   params,
	}

	_, err = UpdateStack(req)

	NotifySuccess("release:promote", map[string]string{
		"app": r.App,
		"id":  r.Id,
	})

	return err
}
Exemple #11
0
func TestManifestMarshalYaml(t *testing.T) {

	strCmd := manifest.Command{
		String: "bin/web",
	}

	arrayCmd := manifest.Command{
		Array: []string{"sh", "-c", "bin/web"},
	}

	m := manifest.Manifest{
		Version: "1",
		Services: map[string]manifest.Service{
			"food": manifest.Service{
				Name: "food",
				Build: manifest.Build{
					Context:    ".",
					Dockerfile: "Dockerfile",
				},
				Command: strCmd,
				Ports: manifest.Ports{
					manifest.Port{
						Public:    true,
						Balancer:  10,
						Container: 10,
					},
				},
			},
		},
	}

	byts, err := yaml.Marshal(m)
	if err != nil {
		t.Error(err.Error())
	}

	m2, err := manifest.Load(byts)
	if err != nil {
		t.Error(err.Error())
	}
	assert.Equal(t, m2.Version, "2")
	assert.Equal(t, m2.Services["food"].Name, "food")
	assert.Equal(t, m2.Services["food"].Command.String, strCmd.String)

	// Test an array Command
	food := m.Services["food"]
	food.Command = arrayCmd
	m.Services["food"] = food

	byts, err = yaml.Marshal(m)
	if err != nil {
		t.Error(err.Error())
	}

	m2, err = manifest.Load(byts)
	if err != nil {
		t.Error(err.Error())
	}
	assert.Equal(t, m2.Version, "2")
	assert.Equal(t, m2.Services["food"].Name, "food")
	assert.Equal(t, m2.Services["food"].Command.Array, arrayCmd.Array)
}
Exemple #12
0
// RunAttached runs a command in the foreground (e.g blocking) and writing the output from said command to rw.
func (a *App) RunAttached(process, command, releaseID string, height, width int, rw io.ReadWriter) error {
	//TODO: A lot of logic in here should be moved to the provider interface.

	resources, err := a.Resources()
	if err != nil {
		return err
	}

	if releaseID == "" {
		releaseID = a.Release
	}

	release, err := GetRelease(a.Name, releaseID)
	if err != nil {
		return err
	}

	var container *ecs.ContainerDefinition
	unpromotedRelease := false

	task, err := ECS().DescribeTaskDefinition(&ecs.DescribeTaskDefinitionInput{
		TaskDefinition: aws.String(resources[UpperName(process)+"ECSTaskDefinition"].Id),
	})
	if err != nil {
		return err
	}

	for _, container = range task.TaskDefinition.ContainerDefinitions {
		if *container.Name == process {
			break
		}
	}

	// This would force an app to be promoted once before being able to run a process.
	if container == nil {
		return fmt.Errorf("unable to find container for %s", process)
	}

	// If the release ID provided does not equal the active one, some logic is needed to determine the next steps.
	// - For a previous release, we iterate over the previous TaskDefinition revisions (starting with the latest) looking for the releaseID specified.
	// - If the release has yet to be promoted, we use the most recent TaskDefinition with the provided release's environment.
	if releaseID != a.Release {

		_, releaseContainer, err := findAppDefinitions(process, releaseID, *task.TaskDefinition.Family, 20)
		if err != nil {
			return err
		}

		// If container is nil, the release most likely hasn't been promoted and thus no TaskDefinition for it.
		if releaseContainer != nil {
			container = releaseContainer

		} else {
			fmt.Printf("Unable to find container for %s. Basing container off of most recent release: %s.\n", process, a.Release)
			unpromotedRelease = true
		}
	}

	var rawEnvs []string
	for _, env := range container.Environment {
		rawEnvs = append(rawEnvs, fmt.Sprintf("%s=%s", *env.Name, *env.Value))
	}
	containerEnvs := structs.Environment{}
	containerEnvs.LoadRaw(strings.Join(rawEnvs, "\n"))

	// Update any environment variables that might be part of the unpromoted release.
	if unpromotedRelease {

		releaseEnv := structs.Environment{}
		releaseEnv.LoadRaw(release.Env)

		for key, value := range releaseEnv {
			containerEnvs[key] = value
		}

		containerEnvs["RELEASE"] = release.Id
	}

	m, err := manifest.Load([]byte(release.Manifest))
	if err != nil {
		return err
	}

	me, ok := m.Services[process]
	if !ok {
		return fmt.Errorf("no such process: %s", process)
	}

	binds := []string{}
	host := ""

	pss, err := ListProcesses(a.Name)
	if err != nil {
		return err
	}

	for _, ps := range pss {
		if ps.Name == process {
			binds = ps.binds
			host = fmt.Sprintf("http://%s:2376", ps.Host)
			break
		}
	}

	var image, repository, tag, username, password, serverAddress string

	if registryId := a.Outputs["RegistryId"]; registryId != "" {
		image = fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s.%s", registryId, os.Getenv("AWS_REGION"), a.Outputs["RegistryRepository"], me.Name, release.Build)
		repository = fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s", registryId, os.Getenv("AWS_REGION"), a.Outputs["RegistryRepository"])
		tag = fmt.Sprintf("%s.%s", me.Name, release.Build)

		resp, err := ECR().GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{
			RegistryIds: []*string{aws.String(a.Outputs["RegistryId"])},
		})

		if err != nil {
			return err
		}

		if len(resp.AuthorizationData) < 1 {
			return fmt.Errorf("no authorization data")
		}

		endpoint := *resp.AuthorizationData[0].ProxyEndpoint
		serverAddress = endpoint[8:]

		data, err := base64.StdEncoding.DecodeString(*resp.AuthorizationData[0].AuthorizationToken)

		if err != nil {
			return err
		}

		parts := strings.SplitN(string(data), ":", 2)

		username = parts[0]
		password = parts[1]
	} else {
		image = fmt.Sprintf("%s/%s-%s:%s", os.Getenv("REGISTRY_HOST"), a.Name, me.Name, release.Build)
		repository = fmt.Sprintf("%s/%s-%s", os.Getenv("REGISTRY_HOST"), a.Name, me.Name)
		tag = release.Build
		serverAddress = os.Getenv("REGISTRY_HOST")
		username = "******"
		password = os.Getenv("PASSWORD")
	}

	d, err := Docker(host)
	if err != nil {
		return err
	}

	err = d.PullImage(docker.PullImageOptions{
		Repository: repository,
		Tag:        tag,
	}, docker.AuthConfiguration{
		ServerAddress: serverAddress,
		Username:      username,
		Password:      password,
	})
	if err != nil {
		return err
	}

	res, err := d.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			AttachStdin:  true,
			AttachStdout: true,
			AttachStderr: true,
			Env:          containerEnvs.List(),
			OpenStdin:    true,
			Tty:          true,
			Cmd:          []string{"sh", "-c", command},
			Image:        image,
			Labels: map[string]string{
				"com.convox.rack.type":    "oneoff",
				"com.convox.rack.app":     a.Name,
				"com.convox.rack.process": process,
				"com.convox.rack.release": release.Id,
			},
		},
		HostConfig: &docker.HostConfig{
			Binds: binds,
		},
	})
	if err != nil {
		return err
	}

	ir, iw := io.Pipe()
	or, ow := io.Pipe()

	go d.AttachToContainer(docker.AttachToContainerOptions{
		Container:    res.ID,
		InputStream:  ir,
		OutputStream: ow,
		ErrorStream:  ow,
		Stream:       true,
		Stdin:        true,
		Stdout:       true,
		Stderr:       true,
		RawTerminal:  true,
	})

	go io.Copy(iw, rw)
	go io.Copy(rw, or)

	// hacky
	time.Sleep(100 * time.Millisecond)

	err = d.StartContainer(res.ID, nil)
	if err != nil {
		return err
	}

	err = d.ResizeContainerTTY(res.ID, height, width)
	if err != nil {
		// In some cases, a container might finish and exit by the time ResizeContainerTTY is called.
		// Resizing the TTY shouldn't cause the call to error out for cases like that.
		fmt.Printf("fn=RunAttached level=warning msg=\"unable to resize container: %s\"", err)
	}

	code, err := d.WaitContainer(res.ID)
	if err != nil {
		return err
	}

	_, err = rw.Write([]byte(fmt.Sprintf("%s%d\n", StatusCodePrefix, code)))
	return err
}
Exemple #13
0
func (p *AWSProvider) generateTaskDefinition(app, process, release string) (*ecs.RegisterTaskDefinitionInput, error) {
	a, err := p.AppGet(app)
	if err != nil {
		return nil, err
	}

	r, err := p.ReleaseGet(app, release)
	if err != nil {
		return nil, err
	}

	m, err := manifest.Load([]byte(r.Manifest))
	if err != nil {
		return nil, err
	}

	s, ok := m.Services[process]
	if !ok {
		return nil, fmt.Errorf("no such process: %s", process)
	}

	rs, err := p.describeStackResources(&cloudformation.DescribeStackResourcesInput{
		StackName: aws.String(fmt.Sprintf("%s-%s", p.Rack, app)),
	})
	if err != nil {
		return nil, err
	}

	sarn := ""
	sn := fmt.Sprintf("Service%s", upperName(process))

	for _, r := range rs.StackResources {
		if *r.LogicalResourceId == sn {
			sarn = *r.PhysicalResourceId
		}
	}
	if sarn == "" {
		return nil, fmt.Errorf("could not find service for process: %s", process)
	}

	sres, err := p.ecs().DescribeServices(&ecs.DescribeServicesInput{
		Cluster:  aws.String(p.Cluster),
		Services: []*string{aws.String(sarn)},
	})
	if err != nil {
		return nil, err
	}
	if len(sres.Services) != 1 {
		return nil, fmt.Errorf("could not look up service for process: %s", process)
	}

	tres, err := p.ecs().DescribeTaskDefinition(&ecs.DescribeTaskDefinitionInput{
		TaskDefinition: sres.Services[0].TaskDefinition,
	})
	if err != nil {
		return nil, err
	}
	if len(tres.TaskDefinition.ContainerDefinitions) < 1 {
		return nil, fmt.Errorf("could not find container definition for process: %s", process)
	}

	senv := map[string]string{}

	for _, e := range tres.TaskDefinition.ContainerDefinitions[0].Environment {
		senv[*e.Name] = *e.Value
	}

	cd := &ecs.ContainerDefinition{
		DockerLabels: map[string]*string{
			"convox.process.type": aws.String("oneoff"),
		},
		Essential:         aws.Bool(true),
		Image:             aws.String(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s.%s", a.Outputs["RegistryId"], p.Region, a.Outputs["RegistryRepository"], process, r.Build)),
		MemoryReservation: aws.Int64(512),
		Name:              aws.String(process),
	}

	if len(s.Command.Array) > 0 {
		cd.Command = make([]*string, len(s.Command.Array))
		for i, c := range s.Command.Array {
			cd.Command[i] = aws.String(c)
		}
	} else if s.Command.String != "" {
		cd.Command = []*string{aws.String("sh"), aws.String("-c"), aws.String(s.Command.String)}
	}

	env := map[string]string{}

	base := map[string]string{
		"APP":        app,
		"AWS_REGION": p.Region,
		"LOG_GROUP":  a.Outputs["LogGroup"],
		"PROCESS":    process,
		"RACK":       p.Rack,
		"RELEASE":    release,
	}

	for k, v := range base {
		env[k] = v
	}

	for k, v := range s.Environment {
		env[k] = v
	}

	for _, e := range strings.Split(r.Env, "\n") {
		p := strings.SplitN(e, "=", 2)
		if len(p) == 2 {
			env[p[0]] = p[1]
		}
	}

	vars := []string{"SCHEME", "USERNAME", "PASSWORD", "HOST", "PORT", "PATH", "URL"}

	for _, l := range s.Links {
		prefix := strings.Replace(strings.ToUpper(l), "-", "_", -1)

		for _, v := range vars {
			k := fmt.Sprintf("%s_%s", prefix, v)

			lv, ok := senv[k]
			if !ok {
				return nil, fmt.Errorf("could not find link var: %s", k)
			}

			env[k] = lv
		}
	}

	keys := []string{}

	for k := range env {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, k := range keys {
		cd.Environment = append(cd.Environment, &ecs.KeyValuePair{
			Name:  aws.String(k),
			Value: aws.String(env[k]),
		})
	}

	req := &ecs.RegisterTaskDefinitionInput{
		ContainerDefinitions: []*ecs.ContainerDefinition{cd},
		Family:               aws.String(fmt.Sprintf("%s-%s-%s", p.Rack, app, process)),
	}

	for i, mv := range s.MountableVolumes() {
		name := fmt.Sprintf("volume-%d", i)

		req.Volumes = append(req.Volumes, &ecs.Volume{
			Name: aws.String(name),
			Host: &ecs.HostVolumeProperties{
				SourcePath: aws.String(mv.Host),
			},
		})

		req.ContainerDefinitions[0].MountPoints = append(req.ContainerDefinitions[0].MountPoints, &ecs.MountPoint{
			SourceVolume:  aws.String(name),
			ContainerPath: aws.String(mv.Container),
			ReadOnly:      aws.Bool(false),
		})
	}

	return req, nil
}
Exemple #14
0
// BuildExport exports a build artifact
func (p *AWSProvider) BuildExport(app, id string, w io.Writer) error {
	log := Logger.At("BuildExport").Start()

	build, err := p.BuildGet(app, id)
	if err != nil {
		log.Error(err)
		return err
	}

	m, err := manifest.Load([]byte(build.Manifest))
	if err != nil {
		log.Error(err)
		return fmt.Errorf("manifest error: %s", err)
	}

	if len(m.Services) < 1 {
		log.Errorf("no services found to export")
		return fmt.Errorf("no services found to export")
	}

	bjson, err := json.MarshalIndent(build, "", "  ")
	if err != nil {
		return err
	}

	gz := gzip.NewWriter(w)
	tw := tar.NewWriter(gz)

	dataHeader := &tar.Header{
		Typeflag: tar.TypeReg,
		Name:     "build.json",
		Mode:     0600,
		Size:     int64(len(bjson)),
	}

	if err := tw.WriteHeader(dataHeader); err != nil {
		log.Error(err)
		return err
	}

	if _, err := tw.Write(bjson); err != nil {
		log.Error(err)
		return err
	}

	repo, err := p.appRepository(build.App)
	if err != nil {
		log.Error(err)
		return err
	}

	if err := p.dockerLogin(); err != nil {
		log.Error(err)
		return err
	}

	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		log.Error(err)
		return err
	}

	defer os.Remove(tmp)

	for service := range m.Services {
		image := fmt.Sprintf("%s:%s.%s", repo.URI, service, build.Id)
		file := filepath.Join(tmp, fmt.Sprintf("%s.%s.tar", service, build.Id))

		log.Step("pull").Logf("image=%q", image)
		out, err := exec.Command("docker", "pull", image).CombinedOutput()
		if err != nil {
			return log.Error(fmt.Errorf("%s: %s\n", lastline(out), err.Error()))
		}

		log.Step("save").Logf("image=%q file=%q", image, file)
		out, err = exec.Command("docker", "save", "-o", file, image).CombinedOutput()
		if err != nil {
			return log.Error(fmt.Errorf("%s: %s\n", lastline(out), err.Error()))
		}

		stat, err := os.Stat(file)
		if err != nil {
			log.Error(err)
			return err
		}

		header := &tar.Header{
			Typeflag: tar.TypeReg,
			Name:     fmt.Sprintf("%s.%s.tar", service, build.Id),
			Mode:     0600,
			Size:     stat.Size(),
		}

		if err := tw.WriteHeader(header); err != nil {
			log.Error(err)
			return err
		}

		fd, err := os.Open(file)
		if err != nil {
			log.Error(err)
			return err
		}

		log.Step("copy").Logf("file=%q", file)
		if _, err := io.Copy(tw, fd); err != nil {
			log.Error(err)
			return err
		}

		if err := os.Remove(file); err != nil {
			log.Error(err)
			return err
		}
	}

	if err := tw.Close(); err != nil {
		log.Error(err)
		return err
	}

	if err := gz.Close(); err != nil {
		log.Error(err)
		return err
	}

	log.Success()
	return nil
}