func (s *ControllerSuite) generateControllerExamples(t *c.C) map[string]interface{} { cmd := exec.Command(exec.DockerImage(imageURIs["controller-examples"]), "/bin/flynn-controller-examples") cmd.Env = map[string]string{ "CONTROLLER_KEY": s.clusterConf(t).Key, "SKIP_MIGRATE_DOMAIN": "true", } var stdout bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr err := cmd.Run() t.Logf("stdout: %q", stdout.String()) t.Logf("stderr: %q", stderr.String()) t.Assert(err, c.IsNil) var controllerExamples map[string]json.RawMessage t.Assert(json.Unmarshal(stdout.Bytes(), &controllerExamples), c.IsNil) examples := make(map[string]interface{}, len(controllerExamples)) for key, data := range controllerExamples { example, err := unmarshalControllerExample(data) t.Assert(err, c.IsNil) examples[key] = example } return examples }
func (m *manifestService) runArtifactJob(dgst digest.Digest) error { url := fmt.Sprintf("http://*****:*****@docker-receive.discoverd?name=%s&id=%s", m.authKey, m.repository.Name(), dgst) cmd := exec.Command(m.artifact, "/bin/docker-artifact", url) cmd.Env = map[string]string{ "CONTROLLER_KEY": os.Getenv("CONTROLLER_KEY"), } cmd.Volumes = []*ct.VolumeReq{{Path: "/tmp", DeleteOnStop: true}} if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("error running artifact job: %s: %s", err, out) } return nil }
func convert(slugbuilder *ct.Artifact, slugURL string) (string, error) { res, err := http.Get(slugURL) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode == http.StatusNotFound { return "", ErrNotFound } else if res.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) } id := random.UUID() cmd := exec.Command(slugbuilder, "/bin/convert-legacy-slug.sh") cmd.Env = map[string]string{ "CONTROLLER_KEY": os.Getenv("CONTROLLER_KEY"), "SLUG_IMAGE_ID": id, } cmd.Stdin = res.Body cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Volumes = []*ct.VolumeReq{{Path: "/tmp", DeleteOnStop: true}} return id, cmd.Run() }
func main() { client, err := controller.NewClient("", os.Getenv("CONTROLLER_AUTH_KEY")) if err != nil { log.Fatalln("Unable to connect to controller:", err) } // TODO: use discoverd http dialer here? services, err := discoverd.Services("shelf", discoverd.DefaultTimeout) if err != nil || len(services) < 1 { log.Fatalf("Unable to discover shelf %q", err) } shelfHost := services[0].Addr app := os.Args[1] commit := os.Args[2] _, err = client.GetApp(app) if err == controller.ErrNotFound { log.Fatalf("Unknown app %q", app) } else if err != nil { log.Fatalln("Error retrieving app:", err) } prevRelease, err := client.GetAppRelease(app) if err == controller.ErrNotFound { prevRelease = &ct.Release{} } else if err != nil { log.Fatalln("Error creating getting current app release:", err) } fmt.Printf("-----> Building %s...\n", app) var output bytes.Buffer slugURL := fmt.Sprintf("http://%s/%s.tgz", shelfHost, commit) cmd := exec.Command("flynn/slugbuilder", slugURL) cmd.Stdout = io.MultiWriter(os.Stdout, &output) cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok { cmd.Env = map[string]string{"BUILDPACK_URL": buildpackURL} } if err := cmd.Run(); err != nil { log.Fatalln("Build failed:", err) } var types []string if match := typesPattern.FindSubmatch(output.Bytes()); match != nil { types = strings.Split(string(match[1]), ", ") } fmt.Printf("-----> Creating release...\n") artifact := &ct.Artifact{URI: "docker://flynn/slugrunner"} if err := client.CreateArtifact(artifact); err != nil { log.Fatalln("Error creating artifact:", err) } release := &ct.Release{ ArtifactID: artifact.ID, Env: prevRelease.Env, } procs := make(map[string]ct.ProcessType) for _, t := range types { proc := prevRelease.Processes[t] proc.Cmd = []string{"start", t} if t == "web" { proc.Ports.TCP = 1 if proc.Env == nil { proc.Env = make(map[string]string) } proc.Env["SD_NAME"] = app + "-web" } procs[t] = proc } release.Processes = procs if release.Env == nil { release.Env = make(map[string]string) } release.Env["SLUG_URL"] = slugURL if err := client.CreateRelease(release); err != nil { log.Fatalln("Error creating release:", err) } if err := client.SetAppRelease(app, release.ID); err != nil { log.Fatalln("Error setting app release:", err) } fmt.Println("=====> Application deployed") }
func main() { client, err := controller.NewClient("", os.Getenv("CONTROLLER_AUTH_KEY")) if err != nil { log.Fatalln("Unable to connect to controller:", err) } appName := os.Args[1] app, err := client.GetApp(appName) if err == controller.ErrNotFound { log.Fatalf("Unknown app %q", appName) } else if err != nil { log.Fatalln("Error retrieving app:", err) } prevRelease, err := client.GetAppRelease(app.Name) if err == controller.ErrNotFound { prevRelease = &ct.Release{} } else if err != nil { log.Fatalln("Error getting current app release:", err) } fmt.Printf("-----> Building %s...\n", app.Name) var output bytes.Buffer slugURL := fmt.Sprintf("%s/%s.tgz", blobstoreURL, random.UUID()) cmd := exec.Command(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), slugURL) cmd.Stdout = io.MultiWriter(os.Stdout, &output) cmd.Stderr = os.Stderr if len(prevRelease.Env) > 0 { stdin, err := cmd.StdinPipe() if err != nil { log.Fatalln(err) } go appendEnvDir(os.Stdin, stdin, prevRelease.Env) } else { cmd.Stdin = os.Stdin } cmd.Env = make(map[string]string) cmd.Env["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID) if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok { cmd.Env["BUILDPACK_URL"] = buildpackURL } if err := cmd.Run(); err != nil { log.Fatalln("Build failed:", err) } var types []string if match := typesPattern.FindSubmatch(output.Bytes()); match != nil { types = strings.Split(string(match[1]), ", ") } fmt.Printf("-----> Creating release...\n") artifact := &ct.Artifact{Type: "docker", URI: os.Getenv("SLUGRUNNER_IMAGE_URI")} if err := client.CreateArtifact(artifact); err != nil { log.Fatalln("Error creating artifact:", err) } release := &ct.Release{ ArtifactID: artifact.ID, Env: prevRelease.Env, } procs := make(map[string]ct.ProcessType) for _, t := range types { proc := prevRelease.Processes[t] proc.Cmd = []string{"start", t} if t == "web" { proc.Ports = []ct.Port{{ Port: 8080, Proto: "tcp", Service: &host.Service{ Name: app.Name + "-web", Create: true, Check: &host.HealthCheck{Type: "tcp"}, }, }} } procs[t] = proc } release.Processes = procs if release.Env == nil { release.Env = make(map[string]string) } release.Env["SLUG_URL"] = slugURL if err := client.CreateRelease(release); err != nil { log.Fatalln("Error creating release:", err) } if err := client.DeployAppRelease(app.Name, release.ID); err != nil { log.Fatalln("Error deploying app release:", err) } fmt.Println("=====> Application deployed") // If the app is new and the web process type exists, // it should scale to one process after the release is created. if _, ok := procs["web"]; ok && prevRelease.ID == "" { formation := &ct.Formation{ AppID: app.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, } if err := client.PutFormation(formation); err != nil { log.Fatalln("Error putting formation:", err) } fmt.Println("=====> Added default web=1 formation") } }
func main() { client, err := controller.NewClient("", os.Getenv("CONTROLLER_AUTH_KEY")) if err != nil { log.Fatalln("Unable to connect to controller:", err) } // TODO: use discoverd http dialer here? services, err := discoverd.Services("blobstore", discoverd.DefaultTimeout) if err != nil || len(services) < 1 { log.Fatalf("Unable to discover blobstore %q", err) } blobstoreHost := services[0].Addr appName := os.Args[1] app, err := client.GetApp(appName) if err == controller.ErrNotFound { log.Fatalf("Unknown app %q", appName) } else if err != nil { log.Fatalln("Error retrieving app:", err) } prevRelease, err := client.GetAppRelease(app.Name) if err == controller.ErrNotFound { prevRelease = &ct.Release{} } else if err != nil { log.Fatalln("Error creating getting current app release:", err) } fmt.Printf("-----> Building %s...\n", app.Name) var output bytes.Buffer slugURL := fmt.Sprintf("http://%s/%s.tgz", blobstoreHost, random.UUID()) cmd := exec.Command(exec.DockerImage("flynn/slugbuilder", os.Getenv("SLUGBUILDER_IMAGE_ID")), slugURL) cmd.Stdout = io.MultiWriter(os.Stdout, &output) cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() if err != nil { log.Fatalln(err) } go appendEnvDir(os.Stdin, stdin, prevRelease.Env) if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok { cmd.Env = map[string]string{"BUILDPACK_URL": buildpackURL} } if err := cmd.Run(); err != nil { log.Fatalln("Build failed:", err) } var types []string if match := typesPattern.FindSubmatch(output.Bytes()); match != nil { types = strings.Split(string(match[1]), ", ") } fmt.Printf("-----> Creating release...\n") artifact := &ct.Artifact{Type: "docker", URI: "https://registry.hub.docker.com/flynn/slugrunner?id=" + os.Getenv("SLUGRUNNER_IMAGE_ID")} if err := client.CreateArtifact(artifact); err != nil { log.Fatalln("Error creating artifact:", err) } release := &ct.Release{ ArtifactID: artifact.ID, Env: prevRelease.Env, } procs := make(map[string]ct.ProcessType) for _, t := range types { proc := prevRelease.Processes[t] proc.Cmd = []string{"start", t} if t == "web" { proc.Ports = []ct.Port{{Proto: "tcp"}} if proc.Env == nil { proc.Env = make(map[string]string) } proc.Env["SD_NAME"] = app.Name + "-web" } procs[t] = proc } release.Processes = procs if release.Env == nil { release.Env = make(map[string]string) } release.Env["SLUG_URL"] = slugURL if err := client.CreateRelease(release); err != nil { log.Fatalln("Error creating release:", err) } if err := client.SetAppRelease(app.Name, release.ID); err != nil { log.Fatalln("Error setting app release:", err) } fmt.Println("=====> Application deployed") // If the app is new and the web process type exists, // it should scale to one process after the release is created. if _, ok := procs["web"]; ok && prevRelease.ID == "" { formation := &ct.Formation{ AppID: app.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, } if err := client.PutFormation(formation); err != nil { log.Fatalln("Error putting formation:", err) } fmt.Println("=====> Added default web=1 formation") } }
func runUpdate(args *docopt.Args) error { log := log15.New() // create and update a TUF client log.Info("initializing TUF client") local, err := tuf.FileLocalStore(args.String["--tuf-db"]) if err != nil { log.Error("error creating local TUF client", "err", err) return err } remote, err := tuf.HTTPRemoteStore(args.String["--repository"], tufHTTPOpts("updater")) if err != nil { log.Error("error creating remote TUF client", "err", err) return err } client := tuf.NewClient(local, remote) if !args.Bool["--is-latest"] { return updateAndExecLatest(args.String["--config-dir"], client, log) } // unlink the current binary if it is a temp file if args.Bool["--is-tempfile"] { os.Remove(os.Args[0]) } // read the TUF db so we can pass it to hosts log.Info("reading TUF database") tufDB, err := ioutil.ReadFile(args.String["--tuf-db"]) if err != nil { log.Error("error reading the TUF database", "err", err) return err } log.Info("getting host list") clusterClient := cluster.NewClient() hosts, err := clusterClient.Hosts() if err != nil { log.Error("error getting host list", "err", err) return err } if len(hosts) == 0 { return errors.New("no hosts found") } log.Info(fmt.Sprintf("updating %d hosts", len(hosts))) // eachHost invokes the given function in a goroutine for each host, // returning an error if any of the functions returns an error. eachHost := func(f func(*cluster.Host, log15.Logger) error) (err error) { errs := make(chan error) for _, h := range hosts { go func(host *cluster.Host) { log := log.New("host", host.ID()) errs <- f(host, log) }(h) } for range hosts { if e := <-errs; e != nil { err = e } } return } var mtx sync.Mutex images := make(map[string]string) log.Info("pulling latest images on all hosts") if err := eachHost(func(host *cluster.Host, log log15.Logger) error { log.Info("pulling images") ch := make(chan *layer.PullInfo) stream, err := host.PullImages( args.String["--repository"], args.String["--driver"], args.String["--root"], version.String(), bytes.NewReader(tufDB), ch, ) if err != nil { log.Error("error pulling images", "err", err) return err } defer stream.Close() for info := range ch { if info.Type == layer.TypeLayer { continue } log.Info("pulled image", "name", info.Repo) imageURI := fmt.Sprintf("%s?name=%s&id=%s", args.String["--repository"], info.Repo, info.ID) mtx.Lock() images[info.Repo] = imageURI mtx.Unlock() } if err := stream.Err(); err != nil { log.Error("error pulling images", "err", err) return err } return nil }); err != nil { return err } var binaries map[string]string log.Info("pulling latest binaries and config on all hosts") if err := eachHost(func(host *cluster.Host, log log15.Logger) error { log.Info("pulling binaries and config") paths, err := host.PullBinariesAndConfig( args.String["--repository"], args.String["--bin-dir"], args.String["--config-dir"], version.String(), bytes.NewReader(tufDB), ) if err != nil { log.Error("error pulling binaries and config", "err", err) return err } mtx.Lock() binaries = paths mtx.Unlock() log.Info("binaries and config pulled successfully") return nil }); err != nil { return err } log.Info("validating binaries") flynnHost, ok := binaries["flynn-host"] if !ok { return fmt.Errorf("missing flynn-host binary") } flynnInit, ok := binaries["flynn-init"] if !ok { return fmt.Errorf("missing flynn-init binary") } flynnNSUmount, ok := binaries["flynn-nsumount"] if !ok { return fmt.Errorf("missing flynn-nsumount binary") } log.Info("updating flynn-host daemon on all hosts") if err := eachHost(func(host *cluster.Host, log log15.Logger) error { // TODO(lmars): handle daemons using custom flags (e.g. --state=/foo) _, err := host.Update( flynnHost, "daemon", "--id", host.ID(), "--flynn-init", flynnInit, "--nsumount", flynnNSUmount, ) if err != nil { log.Error("error updating binaries", "err", err) return err } log.Info("flynn-host updated successfully") return nil }); err != nil { return err } updaterImage, ok := images["flynn/updater"] if !ok { e := "missing flynn/updater image" log.Error(e) return errors.New(e) } imageJSON, err := json.Marshal(images) if err != nil { log.Error("error encoding images", "err", err) return err } // use a flag to determine whether to use a TTY log formatter because actually // assigning a TTY to the job causes reading images via stdin to fail. cmd := exec.Command(exec.DockerImage(updaterImage), fmt.Sprintf("--tty=%t", term.IsTerminal(os.Stdout.Fd()))) cmd.Stdin = bytes.NewReader(imageJSON) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return err } log.Info("update complete") return nil }
func main() { client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY")) if err != nil { log.Fatalln("Unable to connect to controller:", err) } appName := os.Args[1] app, err := client.GetApp(appName) if err == controller.ErrNotFound { log.Fatalf("Unknown app %q", appName) } else if err != nil { log.Fatalln("Error retrieving app:", err) } prevRelease, err := client.GetAppRelease(app.Name) if err == controller.ErrNotFound { prevRelease = &ct.Release{} } else if err != nil { log.Fatalln("Error getting current app release:", err) } fmt.Printf("-----> Building %s...\n", app.Name) var output bytes.Buffer slugURL := fmt.Sprintf("%s/%s.tgz", blobstoreURL, random.UUID()) cmd := exec.Command(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), slugURL) cmd.Stdout = io.MultiWriter(os.Stdout, &output) cmd.Stderr = os.Stderr cmd.Meta = map[string]string{ "flynn-controller.app": app.ID, "flynn-controller.app_name": app.Name, "flynn-controller.release": prevRelease.ID, "flynn-controller.type": "slugbuilder", } if len(prevRelease.Env) > 0 { stdin, err := cmd.StdinPipe() if err != nil { log.Fatalln(err) } go appendEnvDir(os.Stdin, stdin, prevRelease.Env) } else { cmd.Stdin = os.Stdin } cmd.Env = make(map[string]string) cmd.Env["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID) if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok { cmd.Env["BUILDPACK_URL"] = buildpackURL } for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} { if v := os.Getenv(k); v != "" { cmd.Env[k] = v } } if err := cmd.Run(); err != nil { log.Fatalln("Build failed:", err) } var types []string if match := typesPattern.FindSubmatch(output.Bytes()); match != nil { types = strings.Split(string(match[1]), ", ") } fmt.Printf("-----> Creating release...\n") artifact := &ct.Artifact{Type: "docker", URI: os.Getenv("SLUGRUNNER_IMAGE_URI")} if err := client.CreateArtifact(artifact); err != nil { log.Fatalln("Error creating artifact:", err) } release := &ct.Release{ ArtifactID: artifact.ID, Env: prevRelease.Env, } procs := make(map[string]ct.ProcessType) for _, t := range types { proc := prevRelease.Processes[t] proc.Cmd = []string{"start", t} if t == "web" { proc.Ports = []ct.Port{{ Port: 8080, Proto: "tcp", Service: &host.Service{ Name: app.Name + "-web", Create: true, Check: &host.HealthCheck{Type: "tcp"}, }, }} } procs[t] = proc } release.Processes = procs if release.Env == nil { release.Env = make(map[string]string) } release.Env["SLUG_URL"] = slugURL if err := client.CreateRelease(release); err != nil { log.Fatalln("Error creating release:", err) } if err := client.DeployAppRelease(app.Name, release.ID); err != nil { log.Fatalln("Error deploying app release:", err) } fmt.Println("=====> Application deployed") if needsDefaultScale(app.ID, prevRelease.ID, procs, client) { formation := &ct.Formation{ AppID: app.ID, ReleaseID: release.ID, Processes: map[string]int{"web": 1}, } watcher, err := client.WatchJobEvents(app.ID, release.ID) if err != nil { log.Fatalln("Error streaming job events", err) return } defer watcher.Close() if err := client.PutFormation(formation); err != nil { log.Fatalln("Error putting formation:", err) } fmt.Println("=====> Waiting for web job to start...") err = watcher.WaitFor(ct.JobEvents{"web": {"up": 1}}, scaleTimeout, func(e *ct.Job) error { switch e.State { case "up": fmt.Println("=====> Default web formation scaled to 1") case "down", "crashed": return fmt.Errorf("Failed to scale web process type") } return nil }) if err != nil { log.Fatalln(err.Error()) } } }
func runUpdate(args *docopt.Args) error { log := log15.New() // create and update a TUF client log.Info("initializing TUF client") local, err := tuf.FileLocalStore(args.String["--tuf-db"]) if err != nil { log.Error("error creating local TUF client", "err", err) return err } remote, err := tuf.HTTPRemoteStore(args.String["--repository"], tufHTTPOpts("updater")) if err != nil { log.Error("error creating remote TUF client", "err", err) return err } client := tuf.NewClient(local, remote) log.Info("updating TUF data") if _, err := client.Update(); err != nil && !tuf.IsLatestSnapshot(err) { log.Error("error updating TUF client", "err", err) return err } // read the TUF db so we can pass it to hosts log.Info("reading TUF database") tufDB, err := ioutil.ReadFile(args.String["--tuf-db"]) if err != nil { log.Error("error reading the TUF database", "err", err) return err } log.Info("getting host list") clusterClient := cluster.NewClient() hosts, err := clusterClient.Hosts() if err != nil { log.Error("error getting host list", "err", err) return err } if len(hosts) == 0 { return errors.New("no hosts found") } log.Info("pulling images on all hosts") images := make(map[string]string) var imageMtx sync.Mutex hostErrs := make(chan error) for _, h := range hosts { go func(host *cluster.Host) { log := log.New("host", host.ID()) log.Info("connecting to host") log.Info("pulling images") ch := make(chan *layer.PullInfo) stream, err := host.PullImages( args.String["--repository"], args.String["--driver"], args.String["--root"], bytes.NewReader(tufDB), ch, ) if err != nil { log.Error("error pulling images", "err", err) hostErrs <- err return } defer stream.Close() for info := range ch { if info.Type == layer.TypeLayer { continue } log.Info("pulled image", "name", info.Repo) imageURI := fmt.Sprintf("%s?name=%s&id=%s", args.String["--repository"], info.Repo, info.ID) imageMtx.Lock() images[info.Repo] = imageURI imageMtx.Unlock() } hostErrs <- stream.Err() }(h) } var hostErr error for _, h := range hosts { if err := <-hostErrs; err != nil { log.Error("error pulling images", "host", h.ID(), "err", err) hostErr = err continue } log.Info("images pulled successfully", "host", h.ID()) } if hostErr != nil { return hostErr } updaterImage, ok := images["flynn/updater"] if !ok { e := "missing flynn/updater image" log.Error(e) return errors.New(e) } imageJSON, err := json.Marshal(images) if err != nil { log.Error("error encoding images", "err", err) return err } // use a flag to determine whether to use a TTY log formatter because actually // assigning a TTY to the job causes reading images via stdin to fail. cmd := exec.Command(exec.DockerImage(updaterImage), fmt.Sprintf("--tty=%t", term.IsTerminal(os.Stdout.Fd()))) cmd.Stdin = bytes.NewReader(imageJSON) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return err } log.Info("update complete") return nil }