// TestAppEvents checks that streaming events for an app only receives events
// for that particular app.
func (s *ControllerSuite) TestAppEvents(t *c.C) {
client := s.controllerClient(t)
app1, release1 := s.createApp(t)
app2, release2 := s.createApp(t)
// stream events for app1
events := make(chan *ct.Job)
stream, err := client.StreamJobEvents(app1.ID, events)
t.Assert(err, c.IsNil)
defer stream.Close()
runJob := func(appID, releaseID string) {
rwc, err := client.RunJobAttached(appID, &ct.NewJob{
ReleaseID: releaseID,
Cmd: []string{"/bin/true"},
DisableLog: true,
})
t.Assert(err, c.IsNil)
rwc.Close()
}
// generate events for app2 and wait for them
watcher, err := client.WatchJobEvents(app2.ID, release2.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
runJob(app2.ID, release2.ID)
t.Assert(watcher.WaitFor(
ct.JobEvents{"": {ct.JobStateUp: 1, ct.JobStateDown: 1}},
10*time.Second,
func(e *ct.Job) error {
debugf(t, "got %s job event for app2", e.State)
return nil
},
), c.IsNil)
// generate events for app1
runJob(app1.ID, release1.ID)
// check the stream only gets events for app1
for {
select {
case e, ok := <-events:
if !ok {
t.Fatal("unexpected close of job event stream")
}
t.Assert(e.AppID, c.Equals, app1.ID)
debugf(t, "got %s job event for app1", e.State)
if e.State == ct.JobStateDown {
return
}
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for job events for app1")
}
}
}
func (s *ControllerSuite) TestResourceLimitsReleaseJob(t *c.C) {
client := s.controllerClient(t)
app, release := s.createApp(t)
watcher, err := client.WatchJobEvents(app.ID, release.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
t.Assert(client.PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: map[string]int{"resources": 1},
}), c.IsNil)
var jobID string
err = watcher.WaitFor(ct.JobEvents{"resources": {ct.JobStateUp: 1, ct.JobStateDown: 1}}, scaleTimeout, func(e *ct.Job) error {
jobID = e.ID
return nil
})
t.Assert(err, c.IsNil)
log := flynn(t, "/", "-a", app.Name, "log", "--job", jobID, "--raw-output")
assertResourceLimits(t, log.Output)
}
func (s *CLISuite) TestSlugReleaseGarbageCollection(t *c.C) {
client := s.controllerClient(t)
// create app with gc.max_inactive_slug_releases=3
maxInactiveSlugReleases := 3
app := &ct.App{Meta: map[string]string{"gc.max_inactive_slug_releases": strconv.Itoa(maxInactiveSlugReleases)}}
t.Assert(client.CreateApp(app), c.IsNil)
// create an image artifact
imageArtifact := s.createArtifact(t, "test-apps")
// create 5 slug artifacts
tmp, err := ioutil.TempFile("", "squashfs-")
t.Assert(err, c.IsNil)
defer os.Remove(tmp.Name())
defer tmp.Close()
t.Assert(exec.Command("mksquashfs", t.MkDir(), tmp.Name(), "-noappend").Run(), c.IsNil)
slug, err := ioutil.ReadAll(tmp)
t.Assert(err, c.IsNil)
slugHash := sha512.Sum512(slug)
slugs := []string{
"http://blobstore.discoverd/layer/1.squashfs",
"http://blobstore.discoverd/layer/2.squashfs",
"http://blobstore.discoverd/layer/3.squashfs",
"http://blobstore.discoverd/layer/4.squashfs",
"http://blobstore.discoverd/layer/5.squashfs",
}
slugArtifacts := make([]*ct.Artifact, len(slugs))
put := func(url string, data []byte) {
req, err := http.NewRequest("PUT", url, bytes.NewReader(data))
t.Assert(err, c.IsNil)
res, err := http.DefaultClient.Do(req)
t.Assert(err, c.IsNil)
res.Body.Close()
t.Assert(res.StatusCode, c.Equals, http.StatusOK)
}
for i, layerURL := range slugs {
manifest := &ct.ImageManifest{
Type: ct.ImageManifestTypeV1,
Rootfs: []*ct.ImageRootfs{{
Layers: []*ct.ImageLayer{{
ID: strconv.Itoa(i + 1),
Type: ct.ImageLayerTypeSquashfs,
Length: int64(len(slug)),
Hashes: map[string]string{"sha512": hex.EncodeToString(slugHash[:])},
}},
}},
}
data := manifest.RawManifest()
url := fmt.Sprintf("http://blobstore.discoverd/image/%s.json", manifest.ID())
put(url, data)
put(layerURL, slug)
artifact := &ct.Artifact{
Type: ct.ArtifactTypeFlynn,
URI: url,
Meta: map[string]string{"blobstore": "true"},
RawManifest: data,
Hashes: manifest.Hashes(),
Size: int64(len(data)),
LayerURLTemplate: "http://blobstore.discoverd/layer/{id}.squashfs",
}
t.Assert(client.CreateArtifact(artifact), c.IsNil)
slugArtifacts[i] = artifact
}
// create 6 releases, the second being scaled up and having the
// same slug as the third (so prevents the slug being deleted)
releases := make([]*ct.Release, 6)
for i, r := range []struct {
slug *ct.Artifact
active bool
}{
{slugArtifacts[0], false},
{slugArtifacts[1], true},
{slugArtifacts[1], false},
{slugArtifacts[2], false},
{slugArtifacts[3], false},
{slugArtifacts[4], false},
} {
release := &ct.Release{
ArtifactIDs: []string{imageArtifact.ID, r.slug.ID},
Processes: map[string]ct.ProcessType{
"app": {Args: []string{"/bin/pingserv"}, Ports: []ct.Port{{Proto: "tcp"}}},
},
Meta: map[string]string{"git": "true"},
}
t.Assert(client.CreateRelease(release), c.IsNil)
procs := map[string]int{"app": 0}
if r.active {
procs["app"] = 1
}
t.Assert(client.PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: procs,
}), c.IsNil)
releases[i] = release
}
// scale the last release so we can deploy it
lastRelease := releases[len(releases)-1]
watcher, err := client.WatchJobEvents(app.ID, lastRelease.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
t.Assert(client.PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: lastRelease.ID,
Processes: map[string]int{"app": 1},
}), c.IsNil)
t.Assert(watcher.WaitFor(ct.JobEvents{"app": ct.JobUpEvents(1)}, scaleTimeout, nil), c.IsNil)
t.Assert(client.SetAppRelease(app.ID, lastRelease.ID), c.IsNil)
// subscribe to garbage collection events
gcEvents := make(chan *ct.Event)
stream, err := client.StreamEvents(ct.StreamEventsOptions{
AppID: app.ID,
ObjectTypes: []ct.EventType{ct.EventTypeAppGarbageCollection},
}, gcEvents)
t.Assert(err, c.IsNil)
defer stream.Close()
// deploy a new release with the same slug as the last release
timeoutCh := make(chan struct{})
time.AfterFunc(5*time.Minute, func() { close(timeoutCh) })
newRelease := *lastRelease
newRelease.ID = ""
t.Assert(client.CreateRelease(&newRelease), c.IsNil)
t.Assert(client.DeployAppRelease(app.ID, newRelease.ID, timeoutCh), c.IsNil)
// wait for garbage collection
select {
case event, ok := <-gcEvents:
if !ok {
t.Fatalf("event stream closed unexpectedly: %s", stream.Err())
}
var e ct.AppGarbageCollectionEvent
t.Assert(json.Unmarshal(event.Data, &e), c.IsNil)
if e.Error != "" {
t.Fatalf("garbage collection failed: %s", e.Error)
}
case <-time.After(60 * time.Second):
t.Fatal("timed out waiting for garbage collection")
}
// check we have 4 distinct slug releases (so 5 in total, only 3 are
// inactive)
list, err := client.AppReleaseList(app.ID)
t.Assert(err, c.IsNil)
t.Assert(list, c.HasLen, maxInactiveSlugReleases+2)
distinctSlugs := make(map[string]struct{}, len(list))
for _, release := range list {
t.Assert(release.ArtifactIDs, c.HasLen, 2)
distinctSlugs[release.ArtifactIDs[1]] = struct{}{}
}
t.Assert(distinctSlugs, c.HasLen, maxInactiveSlugReleases+1)
// check the first and third releases got deleted, but the rest remain
assertDeleted := func(release *ct.Release, deleted bool) {
_, err := client.GetRelease(release.ID)
if deleted {
t.Assert(err, c.Equals, controller.ErrNotFound)
} else {
t.Assert(err, c.IsNil)
}
}
assertDeleted(releases[0], true)
assertDeleted(releases[1], false)
assertDeleted(releases[2], true)
assertDeleted(releases[3], false)
assertDeleted(releases[4], false)
assertDeleted(releases[5], false)
assertDeleted(&newRelease, false)
// check the first slug got deleted, but the rest remain
s.assertURI(t, slugs[0], http.StatusNotFound)
for i := 1; i < len(slugs); i++ {
s.assertURI(t, slugs[i], http.StatusOK)
}
}
func (s *CLISuite) TestRelease(t *c.C) {
app := s.newCliTestApp(t)
defer app.cleanup()
release := &ct.Release{
ArtifactIDs: []string{s.createArtifact(t, "test-apps").ID},
Env: map[string]string{"GLOBAL": "FOO"},
Processes: map[string]ct.ProcessType{
"echoer": {
Args: []string{"/bin/echoer"},
Env: map[string]string{"ECHOER_ONLY": "BAR"},
},
"env": {
Args: []string{"sh", "-c", "env; while true; do sleep 60; done"},
Env: map[string]string{"ENV_ONLY": "BAZ"},
},
},
}
client := s.controllerClient(t)
t.Assert(client.CreateRelease(release), c.IsNil)
t.Assert(client.SetAppRelease(app.id, release.ID), c.IsNil)
updateFile := filepath.Join(t.MkDir(), "updates.json")
updateJSON := []byte(`{
"processes": {
"echoer": {
"env": {"ECHOER_ONLY": "BAT"}
},
"env": {
"env": {"ENV_UPDATE": "QUUX"}
}
}
}`)
t.Assert(ioutil.WriteFile(updateFile, updateJSON, 0644), c.IsNil)
t.Assert(app.flynn("release", "update", updateFile), Succeeds)
resultJSON := []byte(`{
"env": {"GLOBAL": "FOO"},
"processes": {
"echoer": {
"args": ["/bin/echoer"],
"env": {
"ECHOER_ONLY": "BAT"
}
},
"env": {
"args": ["sh", "-c", "env; while true; do sleep 60; done"],
"env": {
"ENV_ONLY": "BAZ",
"ENV_UPDATE": "QUUX"
}
}
}
}`)
result := &ct.Release{}
t.Assert(json.Unmarshal(resultJSON, &result), c.IsNil)
for typ, proc := range result.Processes {
resource.SetDefaults(&proc.Resources)
result.Processes[typ] = proc
}
release, err := s.controller.GetAppRelease(app.name)
t.Assert(err, c.IsNil)
t.Assert(release.Env, c.DeepEquals, result.Env)
t.Assert(release.Processes, c.DeepEquals, result.Processes)
scaleCmd := app.flynn("scale", "--no-wait", "env=1", "foo=1")
t.Assert(scaleCmd, c.Not(Succeeds))
t.Assert(scaleCmd, OutputContains, "ERROR: unknown process types: \"foo\"")
// create a job watcher for the new release
watcher, err := client.WatchJobEvents(app.name, release.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
scaleCmd = app.flynn("scale", "--no-wait", "env=1")
t.Assert(watcher.WaitFor(ct.JobEvents{"env": {ct.JobStateUp: 1}}, scaleTimeout, nil), c.IsNil)
envLog := app.flynn("log")
t.Assert(envLog, Succeeds)
t.Assert(envLog, SuccessfulOutputContains, "GLOBAL=FOO")
t.Assert(envLog, SuccessfulOutputContains, "ENV_ONLY=BAZ")
t.Assert(envLog, SuccessfulOutputContains, "ENV_UPDATE=QUUX")
t.Assert(envLog, c.Not(SuccessfulOutputContains), "ECHOER_ONLY=BAR")
t.Assert(envLog, c.Not(SuccessfulOutputContains), "ECHOER_UPDATE=BAT")
}
func (s *ReleaseSuite) TestReleaseImages(t *c.C) {
if testCluster == nil {
t.Skip("cannot boot release cluster")
}
// stream script output to t.Log
logReader, logWriter := io.Pipe()
defer logWriter.Close()
go func() {
buf := bufio.NewReader(logReader)
for {
line, err := buf.ReadString('\n')
if err != nil {
return
}
debug(t, line[0:len(line)-1])
}
}()
// boot the release cluster, release components to a blobstore and output the new images.json
releaseCluster := s.addReleaseHosts(t)
buildHost := releaseCluster.Instances[0]
var imagesJSON bytes.Buffer
var script bytes.Buffer
slugImageID := random.UUID()
releaseScript.Execute(&script, struct{ ControllerKey, SlugImageID string }{releaseCluster.ControllerKey, slugImageID})
t.Assert(buildHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: &imagesJSON, Stderr: logWriter}), c.IsNil)
var images map[string]*ct.Artifact
t.Assert(json.Unmarshal(imagesJSON.Bytes(), &images), c.IsNil)
// install Flynn from the blobstore on the vanilla host
blobstoreAddr := buildHost.IP + ":8080"
installHost := releaseCluster.Instances[3]
script.Reset()
installScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr})
var installOutput bytes.Buffer
out := io.MultiWriter(logWriter, &installOutput)
t.Assert(installHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil)
// check the flynn-host version is correct
var hostVersion bytes.Buffer
t.Assert(installHost.Run("flynn-host version", &tc.Streams{Stdout: &hostVersion}), c.IsNil)
t.Assert(strings.TrimSpace(hostVersion.String()), c.Equals, "v20161108.0-test")
// check rebuilt images were downloaded
assertInstallOutput := func(format string, v ...interface{}) {
expected := fmt.Sprintf(format, v...)
if !strings.Contains(installOutput.String(), expected) {
t.Fatalf(`expected install to output %q`, expected)
}
}
for name, image := range images {
assertInstallOutput("pulling %s image", name)
for _, layer := range image.Manifest().Rootfs[0].Layers {
assertInstallOutput("pulling %s layer %s", name, layer.ID)
}
}
// installing on an instance with Flynn running should fail
script.Reset()
installScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr})
installOutput.Reset()
err := buildHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out})
if err == nil || !strings.Contains(installOutput.String(), "ERROR: Flynn is already installed.") {
t.Fatal("expected Flynn install to fail but it didn't")
}
// create a controller client for the release cluster
pin, err := base64.StdEncoding.DecodeString(releaseCluster.ControllerPin)
t.Assert(err, c.IsNil)
client, err := controller.NewClientWithConfig(
"https://"+buildHost.IP,
releaseCluster.ControllerKey,
controller.Config{Pin: pin, Domain: releaseCluster.ControllerDomain},
)
t.Assert(err, c.IsNil)
// deploy a slug based app + Redis resource
slugApp := &ct.App{}
t.Assert(client.CreateApp(slugApp), c.IsNil)
gitreceive, err := client.GetAppRelease("gitreceive")
t.Assert(err, c.IsNil)
imageArtifact, err := client.GetArtifact(gitreceive.Env["SLUGRUNNER_IMAGE_ID"])
t.Assert(err, c.IsNil)
slugArtifact, err := client.GetArtifact(slugImageID)
t.Assert(err, c.IsNil)
resource, err := client.ProvisionResource(&ct.ResourceReq{ProviderID: "redis", Apps: []string{slugApp.ID}})
t.Assert(err, c.IsNil)
release := &ct.Release{
ArtifactIDs: []string{imageArtifact.ID, slugArtifact.ID},
Processes: map[string]ct.ProcessType{"web": {Args: []string{"/runner/init", "bin/http"}}},
Meta: map[string]string{"git": "true"},
Env: resource.Env,
}
t.Assert(client.CreateRelease(release), c.IsNil)
t.Assert(client.SetAppRelease(slugApp.ID, release.ID), c.IsNil)
watcher, err := client.WatchJobEvents(slugApp.ID, release.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
t.Assert(client.PutFormation(&ct.Formation{
AppID: slugApp.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 1},
}), c.IsNil)
err = watcher.WaitFor(ct.JobEvents{"web": {ct.JobStateUp: 1}}, scaleTimeout, nil)
t.Assert(err, c.IsNil)
// run a cluster update from the blobstore
updateHost := releaseCluster.Instances[1]
script.Reset()
updateScript.Execute(&script, map[string]string{"Blobstore": blobstoreAddr, "Discoverd": updateHost.IP + ":1111"})
var updateOutput bytes.Buffer
out = io.MultiWriter(logWriter, &updateOutput)
t.Assert(updateHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil)
// check rebuilt images were downloaded
for name := range images {
for _, host := range releaseCluster.Instances[0:2] {
expected := fmt.Sprintf(`"pulling %s image" host=%s`, name, host.ID)
if !strings.Contains(updateOutput.String(), expected) {
t.Fatalf(`expected update to download %s on host %s`, name, host.ID)
}
}
}
assertImage := func(uri, image string) {
t.Assert(uri, c.Equals, images[image].URI)
}
// check system apps were deployed correctly
for _, app := range updater.SystemApps {
if app.ImageOnly {
continue // we don't deploy ImageOnly updates
}
debugf(t, "checking new %s release is using image %s", app.Name, images[app.Name].URI)
expected := fmt.Sprintf(`"finished deploy of system app" name=%s`, app.Name)
if !strings.Contains(updateOutput.String(), expected) {
t.Fatalf(`expected update to deploy %s`, app.Name)
}
release, err := client.GetAppRelease(app.Name)
t.Assert(err, c.IsNil)
debugf(t, "new %s release ID: %s", app.Name, release.ID)
artifact, err := client.GetArtifact(release.ArtifactIDs[0])
t.Assert(err, c.IsNil)
debugf(t, "new %s artifact: %+v", app.Name, artifact)
assertImage(artifact.URI, app.Name)
}
// check gitreceive has the correct slug env vars
gitreceive, err = client.GetAppRelease("gitreceive")
t.Assert(err, c.IsNil)
for _, name := range []string{"slugbuilder", "slugrunner"} {
artifact, err := client.GetArtifact(gitreceive.Env[strings.ToUpper(name)+"_IMAGE_ID"])
t.Assert(err, c.IsNil)
assertImage(artifact.URI, name)
}
// check slug based app was deployed correctly
release, err = client.GetAppRelease(slugApp.Name)
t.Assert(err, c.IsNil)
imageArtifact, err = client.GetArtifact(release.ArtifactIDs[0])
t.Assert(err, c.IsNil)
assertImage(imageArtifact.URI, "slugrunner")
// check Redis app was deployed correctly
release, err = client.GetAppRelease(resource.Env["FLYNN_REDIS"])
t.Assert(err, c.IsNil)
imageArtifact, err = client.GetArtifact(release.ArtifactIDs[0])
t.Assert(err, c.IsNil)
assertImage(imageArtifact.URI, "redis")
}
func main() {
client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY"))
if err != nil {
log.Fatalln("Unable to connect to controller:", err)
}
appName := os.Args[1]
app, err := client.GetApp(appName)
if err == controller.ErrNotFound {
log.Fatalf("Unknown app %q", appName)
} else if err != nil {
log.Fatalln("Error retrieving app:", err)
}
prevRelease, err := client.GetAppRelease(app.Name)
if err == controller.ErrNotFound {
prevRelease = &ct.Release{}
} else if err != nil {
log.Fatalln("Error getting current app release:", err)
}
fmt.Printf("-----> Building %s...\n", app.Name)
var output bytes.Buffer
slugURL := fmt.Sprintf("%s/%s.tgz", blobstoreURL, random.UUID())
cmd := exec.Command(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), slugURL)
cmd.Stdout = io.MultiWriter(os.Stdout, &output)
cmd.Stderr = os.Stderr
cmd.Meta = map[string]string{
"flynn-controller.app": app.ID,
"flynn-controller.app_name": app.Name,
"flynn-controller.release": prevRelease.ID,
"flynn-controller.type": "slugbuilder",
}
if len(prevRelease.Env) > 0 {
stdin, err := cmd.StdinPipe()
if err != nil {
log.Fatalln(err)
}
go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
} else {
cmd.Stdin = os.Stdin
}
cmd.Env = make(map[string]string)
cmd.Env["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID)
if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
cmd.Env["BUILDPACK_URL"] = buildpackURL
}
for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} {
if v := os.Getenv(k); v != "" {
cmd.Env[k] = v
}
}
if err := cmd.Run(); err != nil {
log.Fatalln("Build failed:", err)
}
var types []string
if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
types = strings.Split(string(match[1]), ", ")
}
fmt.Printf("-----> Creating release...\n")
artifact := &ct.Artifact{Type: "docker", URI: os.Getenv("SLUGRUNNER_IMAGE_URI")}
if err := client.CreateArtifact(artifact); err != nil {
log.Fatalln("Error creating artifact:", err)
}
release := &ct.Release{
ArtifactID: artifact.ID,
Env: prevRelease.Env,
}
procs := make(map[string]ct.ProcessType)
for _, t := range types {
proc := prevRelease.Processes[t]
proc.Cmd = []string{"start", t}
if t == "web" {
proc.Ports = []ct.Port{{
Port: 8080,
Proto: "tcp",
Service: &host.Service{
Name: app.Name + "-web",
Create: true,
Check: &host.HealthCheck{Type: "tcp"},
},
}}
}
procs[t] = proc
}
release.Processes = procs
if release.Env == nil {
release.Env = make(map[string]string)
}
release.Env["SLUG_URL"] = slugURL
if err := client.CreateRelease(release); err != nil {
log.Fatalln("Error creating release:", err)
}
if err := client.DeployAppRelease(app.Name, release.ID); err != nil {
log.Fatalln("Error deploying app release:", err)
}
fmt.Println("=====> Application deployed")
if needsDefaultScale(app.ID, prevRelease.ID, procs, client) {
formation := &ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 1},
}
watcher, err := client.WatchJobEvents(app.ID, release.ID)
if err != nil {
log.Fatalln("Error streaming job events", err)
return
}
defer watcher.Close()
if err := client.PutFormation(formation); err != nil {
log.Fatalln("Error putting formation:", err)
}
fmt.Println("=====> Waiting for web job to start...")
err = watcher.WaitFor(ct.JobEvents{"web": {"up": 1}}, scaleTimeout, func(e *ct.Job) error {
switch e.State {
case "up":
fmt.Println("=====> Default web formation scaled to 1")
case "down", "crashed":
return fmt.Errorf("Failed to scale web process type")
}
return nil
})
if err != nil {
log.Fatalln(err.Error())
}
}
}
func (s *CLISuite) TestSlugReleaseGarbageCollection(t *c.C) {
client := s.controllerClient(t)
// create app with gc.max_inactive_slug_releases=3
maxInactiveSlugReleases := 3
app := &ct.App{Meta: map[string]string{"gc.max_inactive_slug_releases": strconv.Itoa(maxInactiveSlugReleases)}}
t.Assert(client.CreateApp(app), c.IsNil)
// create an image artifact
imageArtifact := &ct.Artifact{Type: host.ArtifactTypeDocker, URI: imageURIs["test-apps"]}
t.Assert(client.CreateArtifact(imageArtifact), c.IsNil)
// create 5 slug artifacts
var slug bytes.Buffer
gz := gzip.NewWriter(&slug)
t.Assert(tar.NewWriter(gz).Close(), c.IsNil)
t.Assert(gz.Close(), c.IsNil)
slugs := []string{
"http://blobstore.discoverd/1/slug.tgz",
"http://blobstore.discoverd/2/slug.tgz",
"http://blobstore.discoverd/3/slug.tgz",
"http://blobstore.discoverd/4/slug.tgz",
"http://blobstore.discoverd/5/slug.tgz",
}
slugArtifacts := make([]*ct.Artifact, len(slugs))
for i, uri := range slugs {
req, err := http.NewRequest("PUT", uri, bytes.NewReader(slug.Bytes()))
t.Assert(err, c.IsNil)
res, err := http.DefaultClient.Do(req)
t.Assert(err, c.IsNil)
res.Body.Close()
t.Assert(res.StatusCode, c.Equals, http.StatusOK)
artifact := &ct.Artifact{
Type: host.ArtifactTypeFile,
URI: uri,
Meta: map[string]string{"blobstore": "true"},
}
t.Assert(client.CreateArtifact(artifact), c.IsNil)
slugArtifacts[i] = artifact
}
// create 6 releases, the second being scaled up and having the
// same slug as the third (so prevents the slug being deleted)
releases := make([]*ct.Release, 6)
for i, r := range []struct {
slug *ct.Artifact
active bool
}{
{slugArtifacts[0], false},
{slugArtifacts[1], true},
{slugArtifacts[1], false},
{slugArtifacts[2], false},
{slugArtifacts[3], false},
{slugArtifacts[4], false},
} {
release := &ct.Release{
ArtifactIDs: []string{imageArtifact.ID, r.slug.ID},
Processes: map[string]ct.ProcessType{
"app": {Args: []string{"/bin/pingserv"}, Ports: []ct.Port{{Proto: "tcp"}}},
},
}
t.Assert(client.CreateRelease(release), c.IsNil)
procs := map[string]int{"app": 0}
if r.active {
procs["app"] = 1
}
t.Assert(client.PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: procs,
}), c.IsNil)
releases[i] = release
}
// scale the last release so we can deploy it
lastRelease := releases[len(releases)-1]
watcher, err := client.WatchJobEvents(app.ID, lastRelease.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
t.Assert(client.PutFormation(&ct.Formation{
AppID: app.ID,
ReleaseID: lastRelease.ID,
Processes: map[string]int{"app": 1},
}), c.IsNil)
t.Assert(watcher.WaitFor(ct.JobEvents{"app": ct.JobUpEvents(1)}, scaleTimeout, nil), c.IsNil)
t.Assert(client.SetAppRelease(app.ID, lastRelease.ID), c.IsNil)
// subscribe to garbage collection events
gcEvents := make(chan *ct.Event)
stream, err := client.StreamEvents(ct.StreamEventsOptions{
AppID: app.ID,
ObjectTypes: []ct.EventType{ct.EventTypeAppGarbageCollection},
}, gcEvents)
t.Assert(err, c.IsNil)
defer stream.Close()
// deploy a new release with the same slug as the last release
timeoutCh := make(chan struct{})
time.AfterFunc(5*time.Minute, func() { close(timeoutCh) })
newRelease := *lastRelease
newRelease.ID = ""
t.Assert(client.CreateRelease(&newRelease), c.IsNil)
t.Assert(client.DeployAppRelease(app.ID, newRelease.ID, timeoutCh), c.IsNil)
// wait for garbage collection
select {
case event, ok := <-gcEvents:
if !ok {
t.Fatalf("event stream closed unexpectedly: %s", stream.Err())
}
var e ct.AppGarbageCollectionEvent
t.Assert(json.Unmarshal(event.Data, &e), c.IsNil)
if e.Error != "" {
t.Fatalf("garbage collection failed: %s", e.Error)
}
case <-time.After(60 * time.Second):
t.Fatal("timed out waiting for garbage collection")
}
// check we have 4 distinct slug releases (so 5 in total, only 3 are
// inactive)
list, err := client.AppReleaseList(app.ID)
t.Assert(err, c.IsNil)
t.Assert(list, c.HasLen, maxInactiveSlugReleases+2)
distinctSlugs := make(map[string]struct{}, len(list))
for _, release := range list {
files := release.FileArtifactIDs()
t.Assert(files, c.HasLen, 1)
distinctSlugs[files[0]] = struct{}{}
}
t.Assert(distinctSlugs, c.HasLen, maxInactiveSlugReleases+1)
// check the first and third releases got deleted, but the rest remain
assertDeleted := func(release *ct.Release, deleted bool) {
_, err := client.GetRelease(release.ID)
if deleted {
t.Assert(err, c.Equals, controller.ErrNotFound)
} else {
t.Assert(err, c.IsNil)
}
}
assertDeleted(releases[0], true)
assertDeleted(releases[1], false)
assertDeleted(releases[2], true)
assertDeleted(releases[3], false)
assertDeleted(releases[4], false)
assertDeleted(releases[5], false)
assertDeleted(&newRelease, false)
// check the first slug got deleted, but the rest remain
s.assertURI(t, slugs[0], http.StatusNotFound)
for i := 1; i < len(slugs); i++ {
s.assertURI(t, slugs[i], http.StatusOK)
}
}
func (s *ReleaseSuite) TestReleaseImages(t *c.C) {
if testCluster == nil {
t.Skip("cannot boot release cluster")
}
// stream script output to t.Log
logReader, logWriter := io.Pipe()
defer logWriter.Close()
go func() {
buf := bufio.NewReader(logReader)
for {
line, err := buf.ReadString('\n')
if err != nil {
return
}
debug(t, line[0:len(line)-1])
}
}()
// boot the release cluster, release components to a blobstore and output the new version.json
releaseCluster := s.addReleaseHosts(t)
buildHost := releaseCluster.Instances[0]
var versionJSON bytes.Buffer
t.Assert(buildHost.Run("bash -ex", &tc.Streams{Stdin: releaseScript, Stdout: &versionJSON, Stderr: logWriter}), c.IsNil)
var versions map[string]string
t.Assert(json.Unmarshal(versionJSON.Bytes(), &versions), c.IsNil)
// install Flynn from the blobstore on the vanilla host
blobstore := struct{ Blobstore string }{buildHost.IP + ":8080"}
installHost := releaseCluster.Instances[3]
var script bytes.Buffer
installScript.Execute(&script, blobstore)
var installOutput bytes.Buffer
out := io.MultiWriter(logWriter, &installOutput)
t.Assert(installHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil)
// check the flynn-host version is correct
var hostVersion bytes.Buffer
t.Assert(installHost.Run("flynn-host version", &tc.Streams{Stdout: &hostVersion}), c.IsNil)
t.Assert(strings.TrimSpace(hostVersion.String()), c.Equals, "v20150131.0-test")
// check rebuilt images were downloaded
for name, id := range versions {
expected := fmt.Sprintf("%s image %s downloaded", name, id)
if !strings.Contains(installOutput.String(), expected) {
t.Fatalf(`expected install to download %s %s`, name, id)
}
}
// installing on an instance with Flynn running should not fail
script.Reset()
installScript.Execute(&script, blobstore)
t.Assert(buildHost.Run("sudo bash -ex", &tc.Streams{Stdin: &script, Stdout: logWriter, Stderr: logWriter}), c.IsNil)
// create a controller client for the release cluster
pin, err := base64.StdEncoding.DecodeString(releaseCluster.ControllerPin)
t.Assert(err, c.IsNil)
client, err := controller.NewClientWithConfig(
"https://"+buildHost.IP,
releaseCluster.ControllerKey,
controller.Config{Pin: pin, Domain: releaseCluster.ControllerDomain},
)
t.Assert(err, c.IsNil)
// deploy a slug based app
slugApp := &ct.App{}
t.Assert(client.CreateApp(slugApp), c.IsNil)
gitreceive, err := client.GetAppRelease("gitreceive")
t.Assert(err, c.IsNil)
imageArtifact := &ct.Artifact{Type: host.ArtifactTypeDocker, URI: gitreceive.Env["SLUGRUNNER_IMAGE_URI"]}
t.Assert(client.CreateArtifact(imageArtifact), c.IsNil)
slugArtifact := &ct.Artifact{Type: host.ArtifactTypeFile, URI: fmt.Sprintf("http://%s:8080/slug.tgz", buildHost.IP)}
t.Assert(client.CreateArtifact(slugArtifact), c.IsNil)
release := &ct.Release{
ArtifactIDs: []string{imageArtifact.ID, slugArtifact.ID},
Processes: map[string]ct.ProcessType{"web": {Cmd: []string{"bin/http"}}},
}
t.Assert(client.CreateRelease(release), c.IsNil)
t.Assert(client.SetAppRelease(slugApp.ID, release.ID), c.IsNil)
watcher, err := client.WatchJobEvents(slugApp.ID, release.ID)
t.Assert(err, c.IsNil)
defer watcher.Close()
t.Assert(client.PutFormation(&ct.Formation{
AppID: slugApp.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 1},
}), c.IsNil)
err = watcher.WaitFor(ct.JobEvents{"web": {ct.JobStateUp: 1}}, scaleTimeout, nil)
t.Assert(err, c.IsNil)
// run a cluster update from the blobstore
updateHost := releaseCluster.Instances[1]
script.Reset()
updateScript.Execute(&script, blobstore)
var updateOutput bytes.Buffer
out = io.MultiWriter(logWriter, &updateOutput)
t.Assert(updateHost.Run("bash -ex", &tc.Streams{Stdin: &script, Stdout: out, Stderr: out}), c.IsNil)
// check rebuilt images were downloaded
for name := range versions {
for _, host := range releaseCluster.Instances[0:2] {
expected := fmt.Sprintf(`"pulled image" host=%s name=%s`, host.ID, name)
if !strings.Contains(updateOutput.String(), expected) {
t.Fatalf(`expected update to download %s on host %s`, name, host.ID)
}
}
}
assertImage := func(uri, image string) {
u, err := url.Parse(uri)
t.Assert(err, c.IsNil)
t.Assert(u.Query().Get("id"), c.Equals, versions[image])
}
// check system apps were deployed correctly
for _, app := range updater.SystemApps {
if app.ImageOnly {
continue // we don't deploy ImageOnly updates
}
if app.Image == "" {
app.Image = "flynn/" + app.Name
}
debugf(t, "checking new %s release is using image %s", app.Name, versions[app.Image])
expected := fmt.Sprintf(`"finished deploy of system app" name=%s`, app.Name)
if !strings.Contains(updateOutput.String(), expected) {
t.Fatalf(`expected update to deploy %s`, app.Name)
}
release, err := client.GetAppRelease(app.Name)
t.Assert(err, c.IsNil)
debugf(t, "new %s release ID: %s", app.Name, release.ID)
artifact, err := client.GetArtifact(release.ImageArtifactID())
t.Assert(err, c.IsNil)
debugf(t, "new %s artifact: %+v", app.Name, artifact)
assertImage(artifact.URI, app.Image)
}
// check gitreceive has the correct slug env vars
gitreceive, err = client.GetAppRelease("gitreceive")
t.Assert(err, c.IsNil)
assertImage(gitreceive.Env["SLUGBUILDER_IMAGE_URI"], "flynn/slugbuilder")
assertImage(gitreceive.Env["SLUGRUNNER_IMAGE_URI"], "flynn/slugrunner")
// check slug based app was deployed correctly
release, err = client.GetAppRelease(slugApp.Name)
t.Assert(err, c.IsNil)
imageArtifact, err = client.GetArtifact(release.ImageArtifactID())
t.Assert(err, c.IsNil)
assertImage(imageArtifact.URI, "flynn/slugrunner")
}
func main() {
client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY"))
if err != nil {
log.Fatalln("Unable to connect to controller:", err)
}
usage := `
Usage: flynn-receiver <app> <rev> [-e <var>=<val>]... [-m <key>=<val>]...
Options:
-e,--env <var>=<val>
-m,--meta <key>=<val>
`[1:]
args, _ := docopt.Parse(usage, nil, true, version.String(), false)
appName := args.String["<app>"]
env, err := parsePairs(args, "--env")
if err != nil {
log.Fatal(err)
}
meta, err := parsePairs(args, "--meta")
if err != nil {
log.Fatal(err)
}
app, err := client.GetApp(appName)
if err == controller.ErrNotFound {
log.Fatalf("Unknown app %q", appName)
} else if err != nil {
log.Fatalln("Error retrieving app:", err)
}
prevRelease, err := client.GetAppRelease(app.Name)
if err == controller.ErrNotFound {
prevRelease = &ct.Release{}
} else if err != nil {
log.Fatalln("Error getting current app release:", err)
}
fmt.Printf("-----> Building %s...\n", app.Name)
jobEnv := make(map[string]string)
jobEnv["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID)
if buildpackURL, ok := env["BUILDPACK_URL"]; ok {
jobEnv["BUILDPACK_URL"] = buildpackURL
} else if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
jobEnv["BUILDPACK_URL"] = buildpackURL
}
for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} {
if v := os.Getenv(k); v != "" {
jobEnv[k] = v
}
}
slugURL := fmt.Sprintf("%s/%s.tgz", blobstoreURL, random.UUID())
cmd := exec.Job(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), &host.Job{
Config: host.ContainerConfig{
Cmd: []string{slugURL},
Env: jobEnv,
Stdin: true,
DisableLog: true,
},
Partition: "background",
Metadata: map[string]string{
"flynn-controller.app": app.ID,
"flynn-controller.app_name": app.Name,
"flynn-controller.release": prevRelease.ID,
"flynn-controller.type": "slugbuilder",
},
})
var output bytes.Buffer
cmd.Stdout = io.MultiWriter(os.Stdout, &output)
cmd.Stderr = os.Stderr
if len(prevRelease.Env) > 0 {
stdin, err := cmd.StdinPipe()
if err != nil {
log.Fatalln(err)
}
go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
} else {
cmd.Stdin = os.Stdin
}
if err := cmd.Run(); err != nil {
log.Fatalln("Build failed:", err)
}
var types []string
if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
types = strings.Split(string(match[1]), ", ")
}
fmt.Printf("-----> Creating release...\n")
artifact := &ct.Artifact{Type: "docker", URI: os.Getenv("SLUGRUNNER_IMAGE_URI")}
if err := client.CreateArtifact(artifact); err != nil {
log.Fatalln("Error creating artifact:", err)
}
release := &ct.Release{
ArtifactID: artifact.ID,
Env: prevRelease.Env,
Meta: prevRelease.Meta,
}
if release.Meta == nil {
release.Meta = make(map[string]string, len(meta))
}
if release.Env == nil {
release.Env = make(map[string]string, len(env))
}
for k, v := range env {
release.Env[k] = v
}
for k, v := range meta {
release.Meta[k] = v
}
procs := make(map[string]ct.ProcessType)
for _, t := range types {
proc := prevRelease.Processes[t]
proc.Cmd = []string{"start", t}
if t == "web" || strings.HasSuffix(t, "-web") {
proc.Service = app.Name + "-" + t
proc.Ports = []ct.Port{{
Port: 8080,
Proto: "tcp",
Service: &host.Service{
Name: proc.Service,
Create: true,
Check: &host.HealthCheck{Type: "tcp"},
},
}}
}
procs[t] = proc
}
release.Processes = procs
if release.Env == nil {
release.Env = make(map[string]string)
}
release.Env["SLUG_URL"] = slugURL
if err := client.CreateRelease(release); err != nil {
log.Fatalln("Error creating release:", err)
}
if err := client.DeployAppRelease(app.Name, release.ID); err != nil {
log.Fatalln("Error deploying app release:", err)
}
fmt.Println("=====> Application deployed")
if needsDefaultScale(app.ID, prevRelease.ID, procs, client) {
formation := &ct.Formation{
AppID: app.ID,
ReleaseID: release.ID,
Processes: map[string]int{"web": 1},
}
watcher, err := client.WatchJobEvents(app.ID, release.ID)
if err != nil {
log.Fatalln("Error streaming job events", err)
return
}
defer watcher.Close()
if err := client.PutFormation(formation); err != nil {
log.Fatalln("Error putting formation:", err)
}
fmt.Println("=====> Waiting for web job to start...")
err = watcher.WaitFor(ct.JobEvents{"web": ct.JobUpEvents(1)}, scaleTimeout, func(e *ct.Job) error {
switch e.State {
case ct.JobStateUp:
fmt.Println("=====> Default web formation scaled to 1")
case ct.JobStateDown:
return fmt.Errorf("Failed to scale web process type")
}
return nil
})
if err != nil {
log.Fatalln(err.Error())
}
}
}