func (s *CLISuite) TestReleaseDelete(t *c.C) { // create an app and release it twice r := s.newGitRepo(t, "http") app := "release-delete-" + random.String(8) t.Assert(r.flynn("create", app), Succeeds) t.Assert(r.git("push", "flynn", "master"), Succeeds) t.Assert(r.git("commit", "--allow-empty", "--message", "empty commit"), Succeeds) t.Assert(r.git("push", "flynn", "master"), Succeeds) // get the releases client := s.controllerClient(t) releases, err := client.AppReleaseList(app) t.Assert(err, c.IsNil) t.Assert(releases, c.HasLen, 2) // check the current release cannot be deleted res := r.flynn("release", "delete", "--yes", releases[0].ID) t.Assert(res, c.Not(Succeeds)) t.Assert(res.Output, c.Equals, "validation_error: cannot delete current app release\n") // associate the initial release with another app otherApp := &ct.App{Name: "release-delete-" + random.String(8)} t.Assert(client.CreateApp(otherApp), c.IsNil) t.Assert(client.PutFormation(&ct.Formation{AppID: otherApp.ID, ReleaseID: releases[1].ID}), c.IsNil) // check deleting the initial release just deletes the formation res = r.flynn("release", "delete", "--yes", releases[1].ID) t.Assert(res, Succeeds) t.Assert(res.Output, c.Equals, "Release scaled down for app but not fully deleted (still associated with 1 other apps)\n") // check the slug artifact still exists slugArtifact, err := client.GetArtifact(releases[1].ArtifactIDs[1]) t.Assert(err, c.IsNil) s.assertURI(t, slugArtifact.URI, http.StatusOK) slugLayerURL := slugArtifact.LayerURL(slugArtifact.Manifest().Rootfs[0].Layers[0]) s.assertURI(t, slugLayerURL, http.StatusOK) // check the inital release can now be deleted res = r.flynn("-a", otherApp.ID, "release", "delete", "--yes", releases[1].ID) t.Assert(res, Succeeds) t.Assert(res.Output, c.Equals, fmt.Sprintf("Deleted release %s (deleted 2 files)\n", releases[1].ID)) // check the slug artifact was deleted _, err = client.GetArtifact(slugArtifact.ID) t.Assert(err, c.Equals, controller.ErrNotFound) s.assertURI(t, slugArtifact.URI, http.StatusNotFound) s.assertURI(t, slugLayerURL, http.StatusNotFound) // check the image artifact was not deleted (since it is shared between both releases) _, err = client.GetArtifact(releases[1].ArtifactIDs[0]) t.Assert(err, c.IsNil) }
// main is a modified version of the registry main function: // https://github.com/docker/distribution/blob/6ba799b/cmd/registry/main.go func main() { logrus.SetLevel(logrus.InfoLevel) ctx := context.Background() ctx = context.WithValue(ctx, "version", version.String()) ctx = context.WithLogger(ctx, context.GetLogger(ctx, "version")) client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY")) if err != nil { context.GetLogger(ctx).Fatalln(err) } release, err := client.GetRelease(os.Getenv("FLYNN_RELEASE_ID")) if err != nil { context.GetLogger(ctx).Fatalln(err) } artifact, err := client.GetArtifact(release.ArtifactIDs[0]) if err != nil { context.GetLogger(ctx).Fatalln(err) } authKey := os.Getenv("AUTH_KEY") middleware.Register("flynn", repositoryMiddleware(client, artifact, authKey)) config := configuration.Configuration{ Version: configuration.CurrentVersion, Storage: configuration.Storage{ blobstore.DriverName: configuration.Parameters{}, "delete": configuration.Parameters{"enabled": true}, }, Middleware: map[string][]configuration.Middleware{ "repository": { {Name: "flynn"}, }, }, Auth: configuration.Auth{ "flynn": configuration.Parameters{ "auth_key": authKey, }, }, } config.HTTP.Secret = os.Getenv("REGISTRY_HTTP_SECRET") status.AddHandler(status.HealthyHandler) app := handlers.NewApp(ctx, config) http.Handle("/", app) addr := ":" + os.Getenv("PORT") context.GetLogger(app).Infof("listening on %s", addr) if err := http.ListenAndServe(addr, nil); err != nil { context.GetLogger(app).Fatalln(err) } }
func (s *CLISuite) TestDockerExportImport(t *c.C) { // release via docker-receive client := s.controllerClient(t) app := &ct.App{Name: "cli-test-docker-export"} t.Assert(client.CreateApp(app), c.IsNil) repo := "cli-test-export" s.buildDockerImage(t, repo, `CMD ["/bin/pingserv"]`) t.Assert(flynn(t, "/", "-a", app.Name, "docker", "push", repo), Succeeds) t.Assert(flynn(t, "/", "-a", app.Name, "scale", "app=1"), Succeeds) defer flynn(t, "/", "-a", app.Name, "scale", "app=0") // export the app file := filepath.Join(t.MkDir(), "export.tar") t.Assert(flynn(t, "/", "-a", app.Name, "export", "-f", file), Succeeds) // delete the image from the registry release, err := client.GetAppRelease(app.Name) t.Assert(err, c.IsNil) artifact, err := client.GetArtifact(release.ImageArtifactID()) t.Assert(err, c.IsNil) u, err := url.Parse(s.clusterConf(t).DockerPushURL) t.Assert(err, c.IsNil) uri := fmt.Sprintf("http://%s/v2/%s/manifests/%s", u.Host, app.Name, artifact.Meta["docker-receive.digest"]) req, err := http.NewRequest("DELETE", uri, nil) req.SetBasicAuth("", s.clusterConf(t).Key) t.Assert(err, c.IsNil) res, err := http.DefaultClient.Do(req) t.Assert(err, c.IsNil) res.Body.Close() // import to another app importApp := "cli-test-docker-import" t.Assert(flynn(t, "/", "import", "--name", importApp, "--file", file), Succeeds) defer flynn(t, "/", "-a", importApp, "scale", "app=0") // wait for it to start _, err = s.discoverdClient(t).Instances(importApp+"-web", 10*time.Second) t.Assert(err, c.IsNil) }
func (s *CLISuite) TestExportImport(t *c.C) { srcApp := "app-export" + random.String(8) dstApp := "app-import" + random.String(8) // create app r := s.newGitRepo(t, "http") t.Assert(r.flynn("create", srcApp), Succeeds) // exporting the app without a release should work file := filepath.Join(t.MkDir(), "export.tar") t.Assert(r.flynn("export", "-f", file), Succeeds) assertExportContains(t, file, "app.json", "routes.json") // exporting the app with an artifact-less release should work t.Assert(r.flynn("env", "set", "FOO=BAR"), Succeeds) t.Assert(r.flynn("export", "-f", file), Succeeds) assertExportContains(t, file, "app.json", "routes.json", "release.json") // release the app and provision some dbs t.Assert(r.git("push", "flynn", "master"), Succeeds) t.Assert(r.flynn("resource", "add", "postgres"), Succeeds) t.Assert(r.flynn("pg", "psql", "--", "-c", "CREATE table foos (data text); INSERT INTO foos (data) VALUES ('foobar')"), Succeeds) t.Assert(r.flynn("resource", "add", "mysql"), Succeeds) t.Assert(r.flynn("mysql", "console", "--", "-e", "CREATE TABLE foos (data TEXT); INSERT INTO foos (data) VALUES ('foobar')"), Succeeds) // grab the slug details client := s.controllerClient(t) release, err := client.GetAppRelease(srcApp) t.Assert(err, c.IsNil) artifact, err := client.GetArtifact(release.ArtifactIDs[1]) t.Assert(err, c.IsNil) slugLayer := artifact.Manifest().Rootfs[0].Layers[0] // export app t.Assert(r.flynn("export", "-f", file), Succeeds) assertExportContains(t, file, "app.json", "routes.json", "release.json", "artifacts.json", slugLayer.ID+".layer", "formation.json", "postgres.dump", "mysql.dump", ) // remove db tables from source app t.Assert(r.flynn("pg", "psql", "--", "-c", "DROP TABLE foos"), Succeeds) t.Assert(r.flynn("mysql", "console", "--", "-e", "DROP TABLE foos"), Succeeds) // remove the git remote t.Assert(r.git("remote", "remove", "flynn"), Succeeds) // import app t.Assert(r.flynn("import", "--name", dstApp, "--file", file), Succeeds) // test dbs were imported query := r.flynn("-a", dstApp, "pg", "psql", "--", "-c", "SELECT * FROM foos") t.Assert(query, SuccessfulOutputContains, "foobar") query = r.flynn("-a", dstApp, "mysql", "console", "--", "-e", "SELECT * FROM foos") t.Assert(query, SuccessfulOutputContains, "foobar") // wait for it to start _, err = s.discoverdClient(t).Instances(dstApp+"-web", 10*time.Second) t.Assert(err, c.IsNil) }
func (s *CLISuite) TestDockerExportImport(t *c.C) { // release via docker-receive client := s.controllerClient(t) app := &ct.App{Name: "cli-test-docker-export"} t.Assert(client.CreateApp(app), c.IsNil) repo := "cli-test-export" s.buildDockerImage(t, repo, `CMD ["/bin/pingserv"]`) t.Assert(flynn(t, "/", "-a", app.Name, "docker", "push", repo), Succeeds) t.Assert(flynn(t, "/", "-a", app.Name, "scale", "app=1"), Succeeds) defer flynn(t, "/", "-a", app.Name, "scale", "app=0") // grab the Flynn image layers release, err := client.GetAppRelease(app.ID) t.Assert(err, c.IsNil) artifact, err := client.GetArtifact(release.ArtifactIDs[0]) t.Assert(err, c.IsNil) layers := artifact.Manifest().Rootfs[0].Layers layerNames := make([]string, len(layers)) for i, layer := range layers { layerNames[i] = layer.ID + ".layer" } // check exporting to stdout works file := filepath.Join(t.MkDir(), "export.tar") cmd := exec.Command("sh", "-c", fmt.Sprintf("%s -a %s export > %s", args.CLI, app.Name, file)) cmd.Env = flynnEnv(flynnrc) var stderr bytes.Buffer cmd.Stderr = &stderr if args.Stream { cmd.Stderr = io.MultiWriter(os.Stderr, &stderr) } if err := cmd.Run(); err != nil { t.Fatalf("error exporting docker app to stdout: %s: %s", err, stderr.String()) } exportFiles := append([]string{ "app.json", "routes.json", "release.json", "artifacts.json", }, append(layerNames, "formation.json")...) assertExportContains(t, file, exportFiles...) // export the app directly to the file t.Assert(flynn(t, "/", "-a", app.Name, "export", "-f", file), Succeeds) assertExportContains(t, file, exportFiles...) // delete the image from the registry u, err := url.Parse(s.clusterConf(t).DockerPushURL) t.Assert(err, c.IsNil) uri := fmt.Sprintf("http://%s/v2/%s/manifests/%s", u.Host, app.Name, artifact.Meta["docker-receive.digest"]) req, err := http.NewRequest("DELETE", uri, nil) req.SetBasicAuth("", s.clusterConf(t).Key) t.Assert(err, c.IsNil) res, err := http.DefaultClient.Do(req) t.Assert(err, c.IsNil) res.Body.Close() // import to another app importApp := "cli-test-docker-import" t.Assert(flynn(t, "/", "import", "--name", importApp, "--file", file), Succeeds) defer flynn(t, "/", "-a", importApp, "scale", "app=0") // wait for it to start _, err = s.discoverdClient(t).Instances(importApp+"-web", 10*time.Second) t.Assert(err, c.IsNil) }
func (s *ReleaseSuite) TestReleaseImages(t *c.C) { if testCluster == nil { t.Skip("cannot boot release cluster") } // stream script output to t.Log logReader, logWriter := io.Pipe() defer logWriter.Close() go func() { buf := bufio.NewReader(logReader) for { line, err := buf.ReadString('\n') if err != nil { return } debug(t, line[0:len(line)-1]) } }() // boot the release cluster, release components to a blobstore and output the new images.json releaseCluster := s.addReleaseHosts(t) buildHost := releaseCluster.Instances[0] var imagesJSON bytes.Buffer var script bytes.Buffer slugImageID := random.UUID() releaseScript.Execute(&script, struct{ ControllerKey, SlugImageID string }{releaseCluster.ControllerKey, slugImageID}) t.Assert(buildHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: &imagesJSON, Stderr: logWriter}), c.IsNil) var images map[string]*ct.Artifact t.Assert(json.Unmarshal(imagesJSON.Bytes(), &images), c.IsNil) // install Flynn from the blobstore on the vanilla host blobstoreAddr := buildHost.IP + ":8080" installHost := releaseCluster.Instances[3] script.Reset() installScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr}) var installOutput bytes.Buffer out := io.MultiWriter(logWriter, &installOutput) t.Assert(installHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check the flynn-host version is correct var hostVersion bytes.Buffer t.Assert(installHost.Run("flynn-host version", &tc.Streams{Stdout: &hostVersion}), c.IsNil) t.Assert(strings.TrimSpace(hostVersion.String()), c.Equals, "v20161108.0-test") // check rebuilt images were downloaded assertInstallOutput := func(format string, v ...interface{}) { expected := fmt.Sprintf(format, v...) if !strings.Contains(installOutput.String(), expected) { t.Fatalf(`expected install to output %q`, expected) } } for name, image := range images { assertInstallOutput("pulling %s image", name) for _, layer := range image.Manifest().Rootfs[0].Layers { assertInstallOutput("pulling %s layer %s", name, layer.ID) } } // installing on an instance with Flynn running should fail script.Reset() installScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr}) installOutput.Reset() err := buildHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}) if err == nil || !strings.Contains(installOutput.String(), "ERROR: Flynn is already installed.") { t.Fatal("expected Flynn install to fail but it didn't") } // create a controller client for the release cluster pin, err := base64.StdEncoding.DecodeString(releaseCluster.ControllerPin) t.Assert(err, c.IsNil) client, err := controller.NewClientWithConfig( "https://"+buildHost.IP, releaseCluster.ControllerKey, controller.Config{Pin: pin, Domain: releaseCluster.ControllerDomain}, ) t.Assert(err, c.IsNil) // deploy a slug based app + Redis resource slugApp := &ct.App{} t.Assert(client.CreateApp(slugApp), c.IsNil) gitreceive, err := client.GetAppRelease("gitreceive") t.Assert(err, c.IsNil) imageArtifact, err := client.GetArtifact(gitreceive.Env["SLUGRUNNER_IMAGE_ID"]) t.Assert(err, c.IsNil) slugArtifact, err := client.GetArtifact(slugImageID) t.Assert(err, c.IsNil) resource, err := client.ProvisionResource(&ct.ResourceReq{ProviderID: "redis", Apps: []string{slugApp.ID}}) t.Assert(err, c.IsNil) release := &ct.Release{ ArtifactIDs: []string{imageArtifact.ID, slugArtifact.ID}, Processes: map[string]ct.ProcessType{"web": {Args: []string{"/runner/init", "bin/http"}}}, Meta: map[string]string{"git": "true"}, Env: resource.Env, } t.Assert(client.CreateRelease(release), c.IsNil) t.Assert(client.SetAppRelease(slugApp.ID, release.ID), c.IsNil) watcher, err := client.WatchJobEvents(slugApp.ID, release.ID) t.Assert(err, c.IsNil) defer watcher.Close() t.Assert(client.PutFormation(&ct.Formation{ AppID: slugApp.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, }), c.IsNil) err = watcher.WaitFor(ct.JobEvents{"web": {ct.JobStateUp: 1}}, scaleTimeout, nil) t.Assert(err, c.IsNil) // run a cluster update from the blobstore updateHost := releaseCluster.Instances[1] script.Reset() updateScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr, "Discoverd": updateHost.IP + ":1111"}) var updateOutput bytes.Buffer out = io.MultiWriter(logWriter, &updateOutput) t.Assert(updateHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check rebuilt images were downloaded for name := range images { for _, host := range releaseCluster.Instances[0:2] { expected := fmt.Sprintf(`"pulling %s image" host=%s`, name, host.ID) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to download %s on host %s`, name, host.ID) } } } assertImage := func(uri, image string) { t.Assert(uri, c.Equals, images[image].URI) } // check system apps were deployed correctly for _, app := range updater.SystemApps { if app.ImageOnly { continue // we don't deploy ImageOnly updates } debugf(t, "checking new %s release is using image %s", app.Name, images[app.Name].URI) expected := fmt.Sprintf(`"finished deploy of system app" name=%s`, app.Name) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to deploy %s`, app.Name) } release, err := client.GetAppRelease(app.Name) t.Assert(err, c.IsNil) debugf(t, "new %s release ID: %s", app.Name, release.ID) artifact, err := client.GetArtifact(release.ArtifactIDs[0]) t.Assert(err, c.IsNil) debugf(t, "new %s artifact: %+v", app.Name, artifact) assertImage(artifact.URI, app.Name) } // check gitreceive has the correct slug env vars gitreceive, err = client.GetAppRelease("gitreceive") t.Assert(err, c.IsNil) for _, name := range []string{"slugbuilder", "slugrunner"} { artifact, err := client.GetArtifact(gitreceive.Env[strings.ToUpper(name)+"_IMAGE_ID"]) t.Assert(err, c.IsNil) assertImage(artifact.URI, name) } // check slug based app was deployed correctly release, err = client.GetAppRelease(slugApp.Name) t.Assert(err, c.IsNil) imageArtifact, err = client.GetArtifact(release.ArtifactIDs[0]) t.Assert(err, c.IsNil) assertImage(imageArtifact.URI, "slugrunner") // check Redis app was deployed correctly release, err = client.GetAppRelease(resource.Env["FLYNN_REDIS"]) t.Assert(err, c.IsNil) imageArtifact, err = client.GetArtifact(release.ArtifactIDs[0]) t.Assert(err, c.IsNil) assertImage(imageArtifact.URI, "redis") }
func (s *ReleaseSuite) TestReleaseImages(t *c.C) { if testCluster == nil { t.Skip("cannot boot release cluster") } // stream script output to t.Log logReader, logWriter := io.Pipe() defer logWriter.Close() go func() { buf := bufio.NewReader(logReader) for { line, err := buf.ReadString('\n') if err != nil { return } debug(t, line[0:len(line)-1]) } }() // boot the release cluster, release components to a blobstore and output the new version.json releaseCluster := s.addReleaseHosts(t) buildHost := releaseCluster.Instances[0] var versionJSON bytes.Buffer t.Assert(buildHost.Run("bash -ex", &tc.Streams{Stdin: releaseScript, Stdout: &versionJSON, Stderr: logWriter}), c.IsNil) var versions map[string]string t.Assert(json.Unmarshal(versionJSON.Bytes(), &versions), c.IsNil) // install Flynn from the blobstore on the vanilla host blobstore := struct{ Blobstore string }{buildHost.IP + ":8080"} installHost := releaseCluster.Instances[3] var script bytes.Buffer installScript.Execute(&script, blobstore) var installOutput bytes.Buffer out := io.MultiWriter(logWriter, &installOutput) t.Assert(installHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check the flynn-host version is correct var hostVersion bytes.Buffer t.Assert(installHost.Run("flynn-host version", &tc.Streams{Stdout: &hostVersion}), c.IsNil) t.Assert(strings.TrimSpace(hostVersion.String()), c.Equals, "v20150131.0-test") // check rebuilt images were downloaded for name, id := range versions { expected := fmt.Sprintf("%s image %s downloaded", name, id) if !strings.Contains(installOutput.String(), expected) { t.Fatalf(`expected install to download %s %s`, name, id) } } // run a cluster update from the blobstore updateHost := releaseCluster.Instances[1] script = bytes.Buffer{} updateScript.Execute(&script, blobstore) var updateOutput bytes.Buffer out = io.MultiWriter(logWriter, &updateOutput) t.Assert(updateHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check rebuilt images were downloaded for name := range versions { for _, host := range releaseCluster.Instances[0:2] { expected := fmt.Sprintf(`"pulled image" host=%s name=%s`, host.ID, name) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to download %s on host %s`, name, host.ID) } } } // create a controller client for the new cluster pin, err := base64.StdEncoding.DecodeString(releaseCluster.ControllerPin) t.Assert(err, c.IsNil) client, err := controller.NewClientWithConfig( "https://"+buildHost.IP, releaseCluster.ControllerKey, controller.Config{Pin: pin, Domain: releaseCluster.ControllerDomain}, ) t.Assert(err, c.IsNil) // check system apps were deployed correctly for _, app := range updater.SystemApps { image := "flynn/" + app if app == "gitreceive" { image = "flynn/receiver" } debugf(t, "checking new %s release is using image %s", app, versions[image]) expected := fmt.Sprintf(`"finished deploy of system app" name=%s`, app) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to deploy %s`, app) } release, err := client.GetAppRelease(app) t.Assert(err, c.IsNil) debugf(t, "new %s release ID: %s", app, release.ID) artifact, err := client.GetArtifact(release.ArtifactID) t.Assert(err, c.IsNil) debugf(t, "new %s artifact: %+v", app, artifact) uri, err := url.Parse(artifact.URI) t.Assert(err, c.IsNil) t.Assert(uri.Query().Get("id"), c.Equals, versions[image]) } }
func (s *ReleaseSuite) TestReleaseImages(t *c.C) { if testCluster == nil { t.Skip("cannot boot release cluster") } // stream script output to t.Log logReader, logWriter := io.Pipe() defer logWriter.Close() go func() { buf := bufio.NewReader(logReader) for { line, err := buf.ReadString('\n') if err != nil { return } debug(t, line[0:len(line)-1]) } }() // boot the release cluster, release components to a blobstore and output the new version.json releaseCluster := s.addReleaseHosts(t) buildHost := releaseCluster.Instances[0] var versionJSON bytes.Buffer t.Assert(buildHost.Run("bash -ex", &tc.Streams{Stdin: releaseScript, Stdout: &versionJSON, Stderr: logWriter}), c.IsNil) var versions map[string]string t.Assert(json.Unmarshal(versionJSON.Bytes(), &versions), c.IsNil) // install Flynn from the blobstore on the vanilla host blobstore := struct{ Blobstore string }{buildHost.IP + ":8080"} installHost := releaseCluster.Instances[3] var script bytes.Buffer installScript.Execute(&script, blobstore) var installOutput bytes.Buffer out := io.MultiWriter(logWriter, &installOutput) t.Assert(installHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check the flynn-host version is correct var hostVersion bytes.Buffer t.Assert(installHost.Run("flynn-host version", &tc.Streams{Stdout: &hostVersion}), c.IsNil) t.Assert(strings.TrimSpace(hostVersion.String()), c.Equals, "v20150131.0-test") // check rebuilt images were downloaded for name, id := range versions { expected := fmt.Sprintf("%s image %s downloaded", name, id) if !strings.Contains(installOutput.String(), expected) { t.Fatalf(`expected install to download %s %s`, name, id) } } // installing on an instance with Flynn running should not fail script.Reset() installScript.Execute(&script, blobstore) t.Assert(buildHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: logWriter, Stderr: logWriter}), c.IsNil) // create a controller client for the release cluster pin, err := base64.StdEncoding.DecodeString(releaseCluster.ControllerPin) t.Assert(err, c.IsNil) client, err := controller.NewClientWithConfig( "https://"+buildHost.IP, releaseCluster.ControllerKey, controller.Config{Pin: pin, Domain: releaseCluster.ControllerDomain}, ) t.Assert(err, c.IsNil) // deploy a slug based app slugApp := &ct.App{} t.Assert(client.CreateApp(slugApp), c.IsNil) gitreceive, err := client.GetAppRelease("gitreceive") t.Assert(err, c.IsNil) imageArtifact := &ct.Artifact{Type: host.ArtifactTypeDocker, URI: gitreceive.Env["SLUGRUNNER_IMAGE_URI"]} t.Assert(client.CreateArtifact(imageArtifact), c.IsNil) slugArtifact := &ct.Artifact{Type: host.ArtifactTypeFile, URI: fmt.Sprintf("http://%s:8080/slug.tgz", buildHost.IP)} t.Assert(client.CreateArtifact(slugArtifact), c.IsNil) release := &ct.Release{ ArtifactIDs: []string{imageArtifact.ID, slugArtifact.ID}, Processes: map[string]ct.ProcessType{"web": {Cmd: []string{"bin/http"}}}, } t.Assert(client.CreateRelease(release), c.IsNil) t.Assert(client.SetAppRelease(slugApp.ID, release.ID), c.IsNil) watcher, err := client.WatchJobEvents(slugApp.ID, release.ID) t.Assert(err, c.IsNil) defer watcher.Close() t.Assert(client.PutFormation(&ct.Formation{ AppID: slugApp.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, }), c.IsNil) err = watcher.WaitFor(ct.JobEvents{"web": {ct.JobStateUp: 1}}, scaleTimeout, nil) t.Assert(err, c.IsNil) // run a cluster update from the blobstore updateHost := releaseCluster.Instances[1] script.Reset() updateScript.Execute(&script, blobstore) var updateOutput bytes.Buffer out = io.MultiWriter(logWriter, &updateOutput) t.Assert(updateHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil) // check rebuilt images were downloaded for name := range versions { for _, host := range releaseCluster.Instances[0:2] { expected := fmt.Sprintf(`"pulled image" host=%s name=%s`, host.ID, name) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to download %s on host %s`, name, host.ID) } } } assertImage := func(uri, image string) { u, err := url.Parse(uri) t.Assert(err, c.IsNil) t.Assert(u.Query().Get("id"), c.Equals, versions[image]) } // check system apps were deployed correctly for _, app := range updater.SystemApps { if app.ImageOnly { continue // we don't deploy ImageOnly updates } if app.Image == "" { app.Image = "flynn/" + app.Name } debugf(t, "checking new %s release is using image %s", app.Name, versions[app.Image]) expected := fmt.Sprintf(`"finished deploy of system app" name=%s`, app.Name) if !strings.Contains(updateOutput.String(), expected) { t.Fatalf(`expected update to deploy %s`, app.Name) } release, err := client.GetAppRelease(app.Name) t.Assert(err, c.IsNil) debugf(t, "new %s release ID: %s", app.Name, release.ID) artifact, err := client.GetArtifact(release.ImageArtifactID()) t.Assert(err, c.IsNil) debugf(t, "new %s artifact: %+v", app.Name, artifact) assertImage(artifact.URI, app.Image) } // check gitreceive has the correct slug env vars gitreceive, err = client.GetAppRelease("gitreceive") t.Assert(err, c.IsNil) assertImage(gitreceive.Env["SLUGBUILDER_IMAGE_URI"], "flynn/slugbuilder") assertImage(gitreceive.Env["SLUGRUNNER_IMAGE_URI"], "flynn/slugrunner") // check slug based app was deployed correctly release, err = client.GetAppRelease(slugApp.Name) t.Assert(err, c.IsNil) imageArtifact, err = client.GetArtifact(release.ImageArtifactID()) t.Assert(err, c.IsNil) assertImage(imageArtifact.URI, "flynn/slugrunner") }
func run() error { client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY")) if err != nil { return fmt.Errorf("Unable to connect to controller: %s", err) } usage := ` Usage: flynn-receiver <app> <rev> [-e <var>=<val>]... [-m <key>=<val>]... Options: -e,--env <var>=<val> -m,--meta <key>=<val> `[1:] args, _ := docopt.Parse(usage, nil, true, version.String(), false) appName := args.String["<app>"] env, err := parsePairs(args, "--env") if err != nil { return err } meta, err := parsePairs(args, "--meta") if err != nil { return err } slugBuilder, err := client.GetArtifact(os.Getenv("SLUGBUILDER_IMAGE_ID")) if err != nil { return fmt.Errorf("Error getting slugbuilder image: %s", err) } slugRunnerID := os.Getenv("SLUGRUNNER_IMAGE_ID") if _, err := client.GetArtifact(slugRunnerID); err != nil { return fmt.Errorf("Error getting slugrunner image: %s", err) } app, err := client.GetApp(appName) if err == controller.ErrNotFound { return fmt.Errorf("Unknown app %q", appName) } else if err != nil { return fmt.Errorf("Error retrieving app: %s", err) } prevRelease, err := client.GetAppRelease(app.Name) if err == controller.ErrNotFound { prevRelease = &ct.Release{} } else if err != nil { return fmt.Errorf("Error getting current app release: %s", err) } fmt.Printf("-----> Building %s...\n", app.Name) slugImageID := random.UUID() jobEnv := map[string]string{ "BUILD_CACHE_URL": fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID), "CONTROLLER_KEY": os.Getenv("CONTROLLER_KEY"), "SLUG_IMAGE_ID": slugImageID, } if buildpackURL, ok := env["BUILDPACK_URL"]; ok { jobEnv["BUILDPACK_URL"] = buildpackURL } else if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok { jobEnv["BUILDPACK_URL"] = buildpackURL } for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} { if v := os.Getenv(k); v != "" { jobEnv[k] = v } } job := &host.Job{ Config: host.ContainerConfig{ Args: []string{"/builder/build.sh"}, Env: jobEnv, Stdin: true, DisableLog: true, }, Partition: "background", Metadata: map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": prevRelease.ID, "flynn-controller.type": "slugbuilder", }, Resources: resource.Defaults(), } if sb, ok := prevRelease.Processes["slugbuilder"]; ok { job.Resources = sb.Resources } else if rawLimit := os.Getenv("SLUGBUILDER_DEFAULT_MEMORY_LIMIT"); rawLimit != "" { if limit, err := resource.ParseLimit(resource.TypeMemory, rawLimit); err == nil { job.Resources[resource.TypeMemory] = resource.Spec{Limit: &limit, Request: &limit} } } cmd := exec.Job(slugBuilder, job) cmd.Volumes = []*ct.VolumeReq{{Path: "/tmp", DeleteOnStop: true}} var output bytes.Buffer cmd.Stdout = io.MultiWriter(os.Stdout, &output) cmd.Stderr = os.Stderr releaseEnv := make(map[string]string, len(env)) if prevRelease.Env != nil { for k, v := range prevRelease.Env { releaseEnv[k] = v } } for k, v := range env { releaseEnv[k] = v } if len(releaseEnv) > 0 { stdin, err := cmd.StdinPipe() if err != nil { return err } go func() { if err := appendEnvDir(os.Stdin, stdin, releaseEnv); err != nil { log.Fatalln("ERROR:", err) } }() } else { cmd.Stdin = os.Stdin } shutdown.BeforeExit(func() { cmd.Kill() }) if err := cmd.Run(); err != nil { return fmt.Errorf("Build failed: %s", err) } var types []string if match := typesPattern.FindSubmatch(output.Bytes()); match != nil { types = strings.Split(string(match[1]), ", ") } fmt.Printf("-----> Creating release...\n") release := &ct.Release{ ArtifactIDs: []string{slugRunnerID, slugImageID}, Env: releaseEnv, Meta: prevRelease.Meta, } if release.Meta == nil { release.Meta = make(map[string]string, len(meta)) } for k, v := range meta { release.Meta[k] = v } procs := make(map[string]ct.ProcessType) for _, t := range types { proc := prevRelease.Processes[t] proc.Args = []string{"/runner/init", "start", t} if (t == "web" || strings.HasSuffix(t, "-web")) && proc.Service == "" { proc.Service = app.Name + "-" + t proc.Ports = []ct.Port{{ Port: 8080, Proto: "tcp", Service: &host.Service{ Name: proc.Service, Create: true, Check: &host.HealthCheck{Type: "tcp"}, }, }} } procs[t] = proc } if sb, ok := prevRelease.Processes["slugbuilder"]; ok { procs["slugbuilder"] = sb } release.Processes = procs if err := client.CreateRelease(release); err != nil { return fmt.Errorf("Error creating release: %s", err) } if err := client.DeployAppRelease(app.Name, release.ID, nil); err != nil { return fmt.Errorf("Error deploying app release: %s", err) } // if the app has a web job and has not been scaled before, create a // web=1 formation and wait for the "APPNAME-web" service to start // (whilst also watching job events so the deploy fails if the job // crashes) if needsDefaultScale(app.ID, prevRelease.ID, procs, client) { fmt.Println("=====> Scaling initial release to web=1") formation := &ct.Formation{ AppID: app.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, } jobEvents := make(chan *ct.Job) jobStream, err := client.StreamJobEvents(app.ID, jobEvents) if err != nil { return fmt.Errorf("Error streaming job events: %s", err) } defer jobStream.Close() serviceEvents := make(chan *discoverd.Event) serviceStream, err := discoverd.NewService(app.Name + "-web").Watch(serviceEvents) if err != nil { return fmt.Errorf("Error streaming service events: %s", err) } defer serviceStream.Close() if err := client.PutFormation(formation); err != nil { return fmt.Errorf("Error putting formation: %s", err) } fmt.Println("-----> Waiting for initial web job to start...") err = func() error { for { select { case e, ok := <-serviceEvents: if !ok { return fmt.Errorf("Service stream closed unexpectedly: %s", serviceStream.Err()) } if e.Kind == discoverd.EventKindUp && e.Instance.Meta["FLYNN_RELEASE_ID"] == release.ID { fmt.Println("=====> Initial web job started") return nil } case e, ok := <-jobEvents: if !ok { return fmt.Errorf("Job stream closed unexpectedly: %s", jobStream.Err()) } if e.State == ct.JobStateDown { return errors.New("Initial web job failed to start") } case <-time.After(time.Duration(app.DeployTimeout) * time.Second): return errors.New("Timed out waiting for initial web job to start") } } }() if err != nil { fmt.Println("-----> WARN: scaling initial release down to web=0 due to error") formation.Processes["web"] = 0 if err := client.PutFormation(formation); err != nil { // just print this error and return the original error fmt.Println("-----> WARN: could not scale the initial release down (it may continue to run):", err) } return err } } fmt.Println("=====> Application deployed") return nil }
func runClusterBackup(args *docopt.Args) error { client, err := getClusterClient() if err != nil { return err } var bar *pb.ProgressBar if term.IsTerminal(os.Stderr.Fd()) { bar = pb.New(0) bar.SetUnits(pb.U_BYTES) bar.ShowBar = false bar.ShowSpeed = true bar.Output = os.Stderr bar.Start() } var dest io.Writer = os.Stdout if filename := args.String["--file"]; filename != "" { f, err := os.Create(filename) if err != nil { return err } defer f.Close() dest = f } fmt.Fprintln(os.Stderr, "Creating cluster backup...") tw := NewTarWriter("flynn-backup-"+time.Now().UTC().Format("2006-01-02_150405"), dest) defer tw.Close() // get app and release details for key apps data := make(map[string]*ct.ExpandedFormation, 4) for _, name := range []string{"postgres", "discoverd", "flannel", "controller"} { app, err := client.GetApp(name) if err != nil { return fmt.Errorf("error getting %s app details: %s", name, err) } release, err := client.GetAppRelease(app.ID) if err != nil { return fmt.Errorf("error getting %s app release: %s", name, err) } formation, err := client.GetFormation(app.ID, release.ID) if err != nil { return fmt.Errorf("error getting %s app formation: %s", name, err) } artifact, err := client.GetArtifact(release.ArtifactID) if err != nil { return fmt.Errorf("error getting %s app artifact: %s", name, err) } data[name] = &ct.ExpandedFormation{ App: app, Release: release, Artifact: artifact, Processes: formation.Processes, } } if err := tw.WriteJSON("flynn.json", data); err != nil { return err } config := &runConfig{ App: "postgres", Release: data["postgres"].Release.ID, Entrypoint: []string{"sh"}, Args: []string{"-c", "pg_dumpall --clean --if-exists | gzip -9"}, Env: map[string]string{ "PGHOST": "leader.postgres.discoverd", "PGUSER": "******", "PGPASSWORD": data["postgres"].Release.Env["PGPASSWORD"], }, DisableLog: true, } if err := tw.WriteCommandOutput(client, "postgres.sql.gz", config, bar); err != nil { return fmt.Errorf("error dumping database: %s", err) } if bar != nil { bar.Finish() } fmt.Fprintln(os.Stderr, "Backup complete.") return nil }