// Fetch an image (or update the local) func (b *DockerBox) Fetch(ctx context.Context, env *util.Environment) (*docker.Image, error) { // TODO(termie): maybe move the container manipulation outside of here? client := b.client e, err := core.EmitterFromContext(ctx) if err != nil { return nil, err } authenticator := b.config.Auth.ToAuthenticator(env) b.repository = authenticator.Repository(env.Interpolate(b.repository)) b.Name = fmt.Sprintf("%s:%s", b.repository, b.tag) // Shortcut to speed up local dev if b.dockerOptions.DockerLocal { image, err := client.InspectImage(env.Interpolate(b.Name)) if err != nil { return nil, err } b.image = image return image, nil } check, err := authenticator.CheckAccess(env.Interpolate(b.repository), auth.Pull) if err != nil { return nil, fmt.Errorf("Error interacting with this repository: %s %v", env.Interpolate(b.repository), err) } if !check { return nil, fmt.Errorf("Not allowed to interact with this repository: %s:", env.Interpolate(b.repository)) } // Create a pipe since we want a io.Reader but Docker expects a io.Writer r, w := io.Pipe() defer w.Close() // emitStatusses in a different go routine go EmitStatus(e, r, b.options) options := docker.PullImageOptions{ // changeme if we have a private registry // Registry: "docker.tsuru.io", OutputStream: w, RawJSONStream: true, Repository: b.repository, Tag: env.Interpolate(b.tag), } authConfig := docker.AuthConfiguration{ Username: authenticator.Username(), Password: authenticator.Password(), } err = client.PullImage(options, authConfig) if err != nil { return nil, err } image, err := client.InspectImage(env.Interpolate(b.Name)) if err != nil { return nil, err } b.image = image return nil, err }
// NewRunner from global options func NewRunner(ctx context.Context, options *core.PipelineOptions, dockerOptions *dockerlocal.DockerOptions, getPipeline pipelineGetter) (*Runner, error) { e, err := core.EmitterFromContext(ctx) if err != nil { return nil, err } logger := util.RootLogger().WithField("Logger", "Runner") // h, err := NewLogHandler() // if err != nil { // p.logger.WithField("Error", err).Panic("Unable to LogHandler") // } // h.ListenTo(e) if options.Debug { dh := core.NewDebugHandler() dh.ListenTo(e) } l, err := event.NewLiteralLogHandler(options) if err != nil { logger.WithField("Error", err).Panic("Unable to event.LiteralLogHandler") } l.ListenTo(e) var mh *event.MetricsEventHandler if options.ShouldKeenMetrics { mh, err = event.NewMetricsHandler(options) if err != nil { logger.WithField("Error", err).Panic("Unable to MetricsHandler") } mh.ListenTo(e) } var r *event.ReportHandler if options.ShouldReport { r, err := event.NewReportHandler(options.ReporterHost, options.ReporterKey) if err != nil { logger.WithField("Error", err).Panic("Unable to event.ReportHandler") } r.ListenTo(e) } return &Runner{ options: options, dockerOptions: dockerOptions, literalLogger: l, metrics: mh, reporter: r, getPipeline: getPipeline, logger: logger, emitter: e, formatter: &util.Formatter{options.GlobalOptions.ShowColors}, }, nil }
// Run executes the service func (b *InternalServiceBox) Run(ctx context.Context, env *util.Environment, links []string) (*docker.Container, error) { e, err := core.EmitterFromContext(ctx) if err != nil { return nil, err } f := &util.Formatter{} client, err := NewDockerClient(b.dockerOptions) if err != nil { return nil, err } // Import the environment and command myEnv := dockerEnv(b.config.Env, env) origEntrypoint := b.image.Config.Entrypoint origCmd := b.image.Config.Cmd cmdInfo := []string{} var entrypoint []string if b.entrypoint != "" { entrypoint, err = shlex.Split(b.entrypoint) if err != nil { return nil, err } cmdInfo = append(cmdInfo, entrypoint...) } else { cmdInfo = append(cmdInfo, origEntrypoint...) } var cmd []string if b.config.Cmd != "" { cmd, err = shlex.Split(b.config.Cmd) if err != nil { return nil, err } cmdInfo = append(cmdInfo, cmd...) } else { cmdInfo = append(cmdInfo, origCmd...) } container, err := client.CreateContainer( docker.CreateContainerOptions{ Name: b.getContainerName(), Config: &docker.Config{ Image: b.Name, Cmd: cmd, Env: myEnv, NetworkDisabled: b.networkDisabled, DNS: b.dockerOptions.DockerDNS, Entrypoint: entrypoint, }, }) if err != nil { return nil, err } out := []string{} for _, part := range cmdInfo { if strings.Contains(part, " ") { out = append(out, fmt.Sprintf("%q", part)) } else { out = append(out, part) } } if b.options.Verbose { b.logger.Println(f.Info(fmt.Sprintf("Starting service %s", b.ShortName), strings.Join(out, " "))) } client.StartContainer(container.ID, &docker.HostConfig{ DNS: b.dockerOptions.DockerDNS, Links: links, }) b.container = container go func() { status, err := client.WaitContainer(container.ID) if err != nil { b.logger.Errorln("Error waiting", err) } b.logger.Debugln("Service container finished with status code:", status, container.ID) if status != 0 { var errstream bytes.Buffer var outstream bytes.Buffer // recv := make(chan string) // outputStream := NewReceiver(recv) opts := docker.LogsOptions{ Container: container.ID, Stdout: true, Stderr: true, ErrorStream: &errstream, OutputStream: &outstream, RawTerminal: false, } err = client.Logs(opts) if err != nil { b.logger.Panicln(err) } e.Emit(core.Logs, &core.LogsArgs{ Stream: fmt.Sprintf("%s-stdout", b.Name), Logs: outstream.String(), }) e.Emit(core.Logs, &core.LogsArgs{ Stream: fmt.Sprintf("%s-stderr", b.Name), Logs: errstream.String(), }) } }() return container, nil }
// Execute commits the current container and pushes it to the configured // registry func (s *DockerPushStep) Execute(ctx context.Context, sess *core.Session) (int, error) { // TODO(termie): could probably re-use the tansport's client client, err := NewDockerClient(s.dockerOptions) if err != nil { return 1, err } e, err := core.EmitterFromContext(ctx) if err != nil { return 1, err } s.logger.WithFields(util.LogFields{ "Registry": s.registry, "Repository": s.repository, "Tags": s.tags, "Message": s.message, }).Debug("Push to registry") // This is clearly only relevant to docker so we're going to dig into the // transport internals a little bit to get the container ID dt := sess.Transport().(*DockerTransport) containerID := dt.containerID auth := docker.AuthConfiguration{ Username: s.username, Password: s.password, Email: s.email, ServerAddress: s.authServer, } if !s.dockerOptions.DockerLocal { checkOpts := CheckAccessOptions{ Auth: auth, Access: "write", Repository: s.repository, Registry: s.registry, } check, err := client.CheckAccess(checkOpts) if err != nil { s.logger.Errorln("Error during check access", err) return -1, err } if !check { s.logger.Errorln("Not allowed to interact with this repository:", s.repository) return -1, fmt.Errorf("Not allowed to interact with this repository: %s", s.repository) } } s.logger.Debugln("Init env:", s.data) config := docker.Config{ Cmd: s.cmd, Entrypoint: s.entrypoint, WorkingDir: s.workingDir, User: s.user, Env: s.env, StopSignal: s.stopSignal, Labels: s.labels, ExposedPorts: s.ports, Volumes: s.volumes, } if len(s.tags) == 0 { s.tags = []string{"latest"} } commitOpts := docker.CommitContainerOptions{ Container: containerID, Repository: s.repository, Author: s.author, Message: s.message, Run: &config, Tag: s.tags[0], } s.logger.Debugln("Commit container:", containerID) i, err := client.CommitContainer(commitOpts) if err != nil { return -1, err } s.logger.WithField("Image", i).Debug("Commit completed") return s.tagAndPush(i.ID, e, client, auth) }
// Execute the scratch-n-push func (s *DockerScratchPushStep) Execute(ctx context.Context, sess *core.Session) (int, error) { // This is clearly only relevant to docker so we're going to dig into the // transport internals a little bit to get the container ID dt := sess.Transport().(*DockerTransport) containerID := dt.containerID _, err := s.CollectArtifact(containerID) if err != nil { return -1, err } // At this point we've written the layer to disk, we're going to add up the // sizes of all the files to add to our json format, and sha256 the data layerFile, err := os.Open(s.options.HostPath("layer.tar")) if err != nil { return -1, err } defer layerFile.Close() var layerSize int64 layerTar := tar.NewReader(layerFile) for { hdr, err := layerTar.Next() if err == io.EOF { // finished the tarball break } if err != nil { return -1, err } // Skip the base dir if hdr.Name == "./" { continue } layerSize += hdr.Size } config := docker.Config{ Cmd: s.cmd, Entrypoint: s.entrypoint, Hostname: containerID[:16], WorkingDir: s.workingDir, ExposedPorts: s.ports, Volumes: s.volumes, } layerID, err := GenerateDockerID() if err != nil { return -1, err } // Make the JSON file we need imageJSON := DockerImageJSON{ Architecture: "amd64", Container: containerID, ContainerConfig: DockerImageJSONContainerConfig{ Hostname: containerID[:16], }, DockerVersion: "1.5", Created: time.Now(), ID: layerID, OS: "linux", Size: layerSize, Config: config, } jsonOut, err := json.MarshalIndent(imageJSON, "", " ") if err != nil { return -1, err } s.logger.Debugln(string(jsonOut)) // Write out the files to disk that we are going to care about err = os.MkdirAll(s.options.HostPath("scratch", layerID), 0755) if err != nil { return -1, err } defer os.RemoveAll(s.options.HostPath("scratch")) // VERSION file versionFile, err := os.OpenFile(s.options.HostPath("scratch", layerID, "VERSION"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer versionFile.Close() _, err = versionFile.Write([]byte("1.0")) if err != nil { return -1, err } err = versionFile.Sync() if err != nil { return -1, err } // json file jsonFile, err := os.OpenFile(s.options.HostPath("scratch", layerID, "json"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer jsonFile.Close() _, err = jsonFile.Write(jsonOut) if err != nil { return -1, err } err = jsonFile.Sync() if err != nil { return -1, err } // repositories file repositoriesFile, err := os.OpenFile(s.options.HostPath("scratch", "repositories"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer repositoriesFile.Close() _, err = repositoriesFile.Write([]byte(fmt.Sprintf(`{"%s":{`, s.repository))) if err != nil { return -1, err } if len(s.tags) == 0 { s.tags = []string{"latest"} } for i, tag := range s.tags { _, err = repositoriesFile.Write([]byte(fmt.Sprintf(`"%s":"%s"`, tag, layerID))) if err != nil { return -1, err } if i != len(s.tags)-1 { _, err = repositoriesFile.Write([]byte{','}) if err != nil { return -1, err } } } _, err = repositoriesFile.Write([]byte{'}', '}'}) err = repositoriesFile.Sync() if err != nil { return -1, err } // layer.tar has an extra folder in it so we have to strip it :/ tempLayerFile, err := os.Open(s.options.HostPath("layer.tar")) if err != nil { return -1, err } defer os.Remove(s.options.HostPath("layer.tar")) defer tempLayerFile.Close() realLayerFile, err := os.OpenFile(s.options.HostPath("scratch", layerID, "layer.tar"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer realLayerFile.Close() tr := tar.NewReader(tempLayerFile) tw := tar.NewWriter(realLayerFile) for { hdr, err := tr.Next() if err == io.EOF { // finished the tarball break } if err != nil { return -1, err } // Skip the base dir if hdr.Name == "./" { continue } if strings.HasPrefix(hdr.Name, "output/") { hdr.Name = hdr.Name[len("output/"):] } else if strings.HasPrefix(hdr.Name, "source/") { hdr.Name = hdr.Name[len("source/"):] } if len(hdr.Name) == 0 { continue } tw.WriteHeader(hdr) _, err = io.Copy(tw, tr) if err != nil { return -1, err } } tw.Close() // Build our output tarball and start writing to it imageFile, err := os.Create(s.options.HostPath("scratch.tar")) defer imageFile.Close() if err != nil { return -1, err } err = util.TarPath(imageFile, s.options.HostPath("scratch")) if err != nil { return -1, err } imageFile.Close() client, err := NewDockerClient(s.dockerOptions) if err != nil { return 1, err } s.logger.WithFields(util.LogFields{ "Registry": s.registry, "Repository": s.repository, "Tags": s.tags, "Message": s.message, }).Debug("Scratch push to registry") // Check the auth auth := docker.AuthConfiguration{ Username: s.username, Password: s.password, Email: s.email, ServerAddress: s.authServer, } if !s.dockerOptions.DockerLocal { checkOpts := CheckAccessOptions{ Auth: auth, Access: "write", Repository: s.repository, Registry: s.registry, } check, err := client.CheckAccess(checkOpts) if err != nil { s.logger.Errorln("Error during check access", err) return -1, err } if !check { s.logger.Errorln("Not allowed to interact with this repository:", s.repository) return -1, fmt.Errorf("Not allowed to interact with this repository: %s", s.repository) } } // Okay, we can access it, do a docker load to import the image then push it loadFile, err := os.Open(s.options.HostPath("scratch.tar")) defer loadFile.Close() err = client.LoadImage(docker.LoadImageOptions{InputStream: loadFile}) if err != nil { return -1, err } e, err := core.EmitterFromContext(ctx) return s.tagAndPush(layerID, e, client, auth) }
func executePipeline(cmdCtx context.Context, options *core.PipelineOptions, dockerOptions *dockerlocal.DockerOptions, getter pipelineGetter) (*RunnerShared, error) { // Boilerplate soft := NewSoftExit(options.GlobalOptions) logger := util.RootLogger().WithField("Logger", "Main") e, err := core.EmitterFromContext(cmdCtx) if err != nil { return nil, err } f := &util.Formatter{options.GlobalOptions.ShowColors} // Set up the runner r, err := NewRunner(cmdCtx, options, dockerOptions, getter) if err != nil { return nil, err } // Main timer mainTimer := util.NewTimer() timer := util.NewTimer() // These will be emitted at the end of the execution, we're going to be // pessimistic and report that we failed, unless overridden at the end of the // execution. fullPipelineFinisher := r.StartFullPipeline(options) pipelineArgs := &core.FullPipelineFinishedArgs{} defer fullPipelineFinisher.Finish(pipelineArgs) buildFinisher := r.StartBuild(options) buildFinishedArgs := &core.BuildFinishedArgs{Box: nil, Result: "failed"} defer buildFinisher.Finish(buildFinishedArgs) // Debug information DumpOptions(options) // Do some sanity checks before starting err = dockerlocal.RequireDockerEndpoint(dockerOptions) if err != nil { return nil, soft.Exit(err) } // Start copying code logger.Println(f.Info("Executing pipeline")) timer.Reset() _, err = r.EnsureCode() if err != nil { e.Emit(core.Logs, &core.LogsArgs{ Stream: "stderr", Logs: err.Error() + "\n", }) return nil, soft.Exit(err) } err = r.CleanupOldBuilds() if err != nil { e.Emit(core.Logs, &core.LogsArgs{ Stream: "stderr", Logs: err.Error() + "\n", }) } if options.Verbose { logger.Printf(f.Success("Copied working dir", timer.String())) } // Setup environment is still a fairly special step, it needs // to start our boxes and get everything set up logger.Println(f.Info("Running step", "setup environment")) timer.Reset() shared, err := r.SetupEnvironment(cmdCtx) if shared.box != nil { if options.ShouldRemove { defer shared.box.Clean() } defer shared.box.Stop() } if err != nil { logger.Errorln(f.Fail("Step failed", "setup environment", timer.String())) e.Emit(core.Logs, &core.LogsArgs{ Stream: "stderr", Logs: err.Error() + "\n", }) return nil, soft.Exit(err) } if options.Verbose { logger.Printf(f.Success("Step passed", "setup environment", timer.String())) } // Expand our context object box := shared.box buildFinishedArgs.Box = box pipeline := shared.pipeline repoName := pipeline.DockerRepo() tag := pipeline.DockerTag() message := pipeline.DockerMessage() shouldStore := options.ShouldArtifacts // TODO(termie): hack for now, probably can be made into a naive class var storeStep core.Step if shouldStore { storeStep = &core.ExternalStep{ BaseStep: core.NewBaseStep(core.BaseStepOptions{ Name: "store", Owner: "wercker", Version: util.Version(), }), } } e.Emit(core.BuildStepsAdded, &core.BuildStepsAddedArgs{ Build: pipeline, Steps: pipeline.Steps(), StoreStep: storeStep, AfterSteps: pipeline.AfterSteps(), }) pr := &core.PipelineResult{ Success: true, FailedStepName: "", FailedStepMessage: "", } // stepCounter starts at 3, step 1 is "get code", step 2 is "setup // environment". stepCounter := &util.Counter{Current: 3} checkpoint := false for _, step := range pipeline.Steps() { // we always want to run the wercker-init step to provide some functions if !checkpoint && stepCounter.Current > 3 { if options.EnableDevSteps && options.Checkpoint != "" { logger.Printf(f.Info("Skipping step", step.DisplayName())) // start at the one after the checkpoint if step.Checkpoint() == options.Checkpoint { logger.Printf(f.Info("Found checkpoint", options.Checkpoint)) checkpoint = true } stepCounter.Increment() continue } } logger.Printf(f.Info("Running step", step.DisplayName())) timer.Reset() sr, err := r.RunStep(shared, step, stepCounter.Increment()) if err != nil { pr.Success = false pr.FailedStepName = step.DisplayName() pr.FailedStepMessage = sr.Message logger.Printf(f.Fail("Step failed", step.DisplayName(), timer.String())) break } if options.EnableDevSteps && step.Checkpoint() != "" { logger.Printf(f.Info("Checkpointing", step.Checkpoint())) box.Commit(box.Repository(), fmt.Sprintf("w-%s", step.Checkpoint()), "checkpoint", false) } if options.Verbose { logger.Printf(f.Success("Step passed", step.DisplayName(), timer.String())) } } if options.ShouldCommit { _, err = box.Commit(repoName, tag, message, true) if err != nil { logger.Errorln("Failed to commit:", err.Error()) } } // We need to wind the counter to where it should be if we failed a step // so that is the number of steps + get code + setup environment + store // TODO(termie): remove all the this "order" stuff completely stepCounter.Current = len(pipeline.Steps()) + 3 if pr.Success && options.ShouldArtifacts { // At this point the build has effectively passed but we can still mess it // up by being unable to deliver the artifacts err = func() error { sr := &StepResult{ Success: false, Artifact: nil, Message: "", PackageURL: "", ExitCode: 1, } finisher := r.StartStep(shared, storeStep, stepCounter.Increment()) defer finisher.Finish(sr) pr.FailedStepName = storeStep.Name() pr.FailedStepMessage = "Unable to store pipeline output" e.Emit(core.Logs, &core.LogsArgs{ Logs: "Storing artifacts\n", }) artifact, err := pipeline.CollectArtifact(shared.containerID) // Ignore ErrEmptyTarball errors if err != util.ErrEmptyTarball { if err != nil { sr.Message = err.Error() e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Storing artifacts failed: %s\n", sr.Message), }) return err } e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Collecting files from %s\n", artifact.GuestPath), }) ignoredDirectories := []string{".git", "node_modules", "vendor", "site-packages"} nameEmit := func(path string, info os.FileInfo, err error) error { relativePath := strings.TrimPrefix(path, artifact.HostPath) if info.IsDir() { if util.ContainsString(ignoredDirectories, info.Name()) { e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf(".%s/ (content omitted)\n", relativePath), }) return filepath.SkipDir } return nil } e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf(".%s\n", relativePath), }) return nil } err = filepath.Walk(artifact.HostPath, nameEmit) if err != nil { sr.Message = err.Error() e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Storing artifacts failed: %s\n", sr.Message), }) return err } tarInfo, err := os.Stat(artifact.HostTarPath) if err != nil { if os.IsNotExist(err) { e.Emit(core.Logs, &core.LogsArgs{ Logs: "No artifacts stored", }) } else { sr.Message = err.Error() e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Storing artifacts failed: %s\n", sr.Message), }) return err } } else { size, unit := util.ConvertUnit(tarInfo.Size()) e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Total artifact size: %d %s\n", size, unit), }) } if options.ShouldStoreS3 { artificer := dockerlocal.NewArtificer(options, dockerOptions) err = artificer.Upload(artifact) if err != nil { sr.Message = err.Error() e.Emit(core.Logs, &core.LogsArgs{ Logs: fmt.Sprintf("Storing artifacts failed: %s\n", sr.Message), }) return err } } sr.PackageURL = artifact.URL() } else { e.Emit(core.Logs, &core.LogsArgs{ Logs: "No artifacts found\n", }) } e.Emit(core.Logs, &core.LogsArgs{ Logs: "Storing artifacts complete\n", }) sr.Success = true sr.ExitCode = 0 return nil }() if err != nil { pr.Success = false logger.WithField("Error", err).Error("Unable to store pipeline output") } } else { stepCounter.Increment() } // We're sending our build finished but we're not done yet, // now is time to run after-steps if we have any if pr.Success { logger.Println(f.Success("Steps passed", mainTimer.String())) buildFinishedArgs.Result = "passed" } buildFinisher.Finish(buildFinishedArgs) pipelineArgs.MainSuccessful = pr.Success if len(pipeline.AfterSteps()) == 0 { // We're about to end the build, so pull the cache and explode it // into the CacheDir if !options.DirectMount { timer.Reset() err = pipeline.CollectCache(shared.containerID) if err != nil { logger.WithField("Error", err).Error("Unable to store cache") } if options.Verbose { logger.Printf(f.Success("Exported Cache", timer.String())) } } if pr.Success { logger.Println(f.Success("Pipeline finished", mainTimer.String())) } else { logger.Println(f.Fail("Pipeline failed", mainTimer.String())) } if !pr.Success { return nil, fmt.Errorf("Step failed: %s", pr.FailedStepName) } return shared, nil } pipelineArgs.RanAfterSteps = true logger.Println(f.Info("Starting after-steps")) // The container may have died, either way we'll have a fresh env container, err := box.Restart() if err != nil { logger.Panicln(err) } newSessCtx, newSess, err := r.GetSession(cmdCtx, container.ID) if err != nil { logger.Panicln(err) } newShared := &RunnerShared{ box: shared.box, pipeline: shared.pipeline, sess: newSess, sessionCtx: newSessCtx, containerID: shared.containerID, config: shared.config, } // Set up the base environment err = pipeline.ExportEnvironment(newSessCtx, newSess) if err != nil { return nil, err } // Add the After-Step parts err = pr.ExportEnvironment(newSessCtx, newSess) if err != nil { return nil, err } for _, step := range pipeline.AfterSteps() { logger.Println(f.Info("Running after-step", step.DisplayName())) timer.Reset() _, err := r.RunStep(newShared, step, stepCounter.Increment()) if err != nil { logger.Println(f.Fail("After-step failed", step.DisplayName(), timer.String())) break } logger.Println(f.Success("After-step passed", step.DisplayName(), timer.String())) } // We're about to end the build, so pull the cache and explode it // into the CacheDir if !options.DirectMount { timer.Reset() err = pipeline.CollectCache(newShared.containerID) if err != nil { logger.WithField("Error", err).Error("Unable to store cache") } if options.Verbose { logger.Printf(f.Success("Exported Cache", timer.String())) } } if pr.Success { logger.Println(f.Success("Pipeline finished", mainTimer.String())) } else { logger.Println(f.Fail("Pipeline failed", mainTimer.String())) } if !pr.Success { return nil, fmt.Errorf("Step failed: %s", pr.FailedStepName) } pipelineArgs.AfterStepSuccessful = pr.Success return shared, nil }
// Fetch an image (or update the local) func (b *DockerBox) Fetch(ctx context.Context, env *util.Environment) (*docker.Image, error) { // TODO(termie): maybe move the container manipulation outside of here? client := b.client e, err := core.EmitterFromContext(ctx) if err != nil { return nil, err } // Shortcut to speed up local dev if b.dockerOptions.DockerLocal { image, err := client.InspectImage(env.Interpolate(b.Name)) if err != nil { return nil, err } b.image = image return image, nil } // Check for access to this image auth := docker.AuthConfiguration{ Username: env.Interpolate(b.config.Username), Password: env.Interpolate(b.config.Password), } checkOpts := CheckAccessOptions{ Auth: auth, Access: "read", Repository: env.Interpolate(b.repository), Registry: env.Interpolate(b.config.Registry), } check, err := client.CheckAccess(checkOpts) if err != nil { b.logger.Errorln("Error during check access") return nil, err } if !check { b.logger.Errorln("Not allowed to interact with this repository:", b.repository) return nil, fmt.Errorf("Not allowed to interact with this repository: %s", b.repository) } // Create a pipe since we want a io.Reader but Docker expects a io.Writer r, w := io.Pipe() defer w.Close() // emitStatusses in a different go routine go EmitStatus(e, r, b.options) options := docker.PullImageOptions{ // changeme if we have a private registry // Registry: "docker.tsuru.io", OutputStream: w, RawJSONStream: true, Repository: env.Interpolate(b.repository), Tag: env.Interpolate(b.tag), } err = client.PullImage(options, auth) if err != nil { return nil, err } image, err := client.InspectImage(env.Interpolate(b.Name)) if err != nil { return nil, err } b.image = image return nil, err }
// Execute does the actual export and upload of the container func (s *StoreContainerStep) Execute(ctx context.Context, sess *core.Session) (int, error) { e, err := core.EmitterFromContext(ctx) if err != nil { return -1, err } // TODO(termie): could probably re-use the tansport's client client, err := NewDockerClient(s.dockerOptions) if err != nil { return -1, err } // This is clearly only relevant to docker so we're going to dig into the // transport internals a little bit to get the container ID dt := sess.Transport().(*DockerTransport) containerID := dt.containerID repoName := s.DockerRepo() tag := s.DockerTag() message := s.DockerMessage() commitOpts := docker.CommitContainerOptions{ Container: containerID, Repository: repoName, Tag: tag, Author: "wercker", Message: message, } s.logger.Debugln("Commit container:", containerID) i, err := client.CommitContainer(commitOpts) if err != nil { return -1, err } s.logger.WithField("Image", i).Debug("Commit completed") e.Emit(core.Logs, &core.LogsArgs{ Logs: "Exporting container\n", }) file, err := ioutil.TempFile(s.options.BuildPath(), "export-image-") if err != nil { s.logger.WithField("Error", err).Error("Unable to create temporary file") return -1, err } hash := sha256.New() w := snappystream.NewWriter(io.MultiWriter(file, hash)) exportImageOptions := docker.ExportImageOptions{ Name: repoName, OutputStream: w, } err = client.ExportImage(exportImageOptions) if err != nil { s.logger.WithField("Error", err).Error("Unable to export image") return -1, err } // Copy is done now, so close temporary file and set the calculatedHash file.Close() calculatedHash := hex.EncodeToString(hash.Sum(nil)) s.logger.WithFields(util.LogFields{ "SHA256": calculatedHash, "TemporaryLocation": file.Name(), }).Println("Export image successful") key := core.GenerateBaseKey(s.options) key = fmt.Sprintf("%s/%s", key, "docker.tar.sz") s.artifact = &core.Artifact{ HostPath: file.Name(), Key: key, Bucket: s.options.S3Bucket, ContentType: "application/x-snappy-framed", Meta: map[string]*string{ "Sha256": &calculatedHash, }, } return 0, nil }
// Execute the scratch-n-push func (s *DockerScratchPushStep) Execute(ctx context.Context, sess *core.Session) (int, error) { // This is clearly only relevant to docker so we're going to dig into the // transport internals a little bit to get the container ID dt := sess.Transport().(*DockerTransport) containerID := dt.containerID _, err := s.CollectArtifact(containerID) if err != nil { return -1, err } // layer.tar has an extra folder in it so we have to strip it :/ artifactReader, err := os.Open(s.options.HostPath("layer.tar")) if err != nil { return -1, err } defer artifactReader.Close() layerFile, err := os.OpenFile(s.options.HostPath("real_layer.tar"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer layerFile.Close() dgst := digest.Canonical.New() mwriter := io.MultiWriter(layerFile, dgst.Hash()) tr := tar.NewReader(artifactReader) tw := tar.NewWriter(mwriter) for { hdr, err := tr.Next() if err == io.EOF { // finished the tarball break } if err != nil { return -1, err } // Skip the base dir if hdr.Name == "./" { continue } if strings.HasPrefix(hdr.Name, "output/") { hdr.Name = hdr.Name[len("output/"):] } else if strings.HasPrefix(hdr.Name, "source/") { hdr.Name = hdr.Name[len("source/"):] } if len(hdr.Name) == 0 { continue } tw.WriteHeader(hdr) _, err = io.Copy(tw, tr) if err != nil { return -1, err } } digest := dgst.Digest() config := &container.Config{ Cmd: s.cmd, Entrypoint: s.entrypoint, Hostname: containerID[:16], WorkingDir: s.workingDir, Volumes: s.volumes, ExposedPorts: tranformPorts(s.ports), } // Make the JSON file we need t := time.Now() base := image.V1Image{ Architecture: "amd64", Container: containerID, ContainerConfig: container.Config{ Hostname: containerID[:16], }, DockerVersion: "1.10", Created: t, OS: "linux", Config: config, } imageJSON := image.Image{ V1Image: base, History: []image.History{image.History{Created: t}}, RootFS: &image.RootFS{ Type: "layers", DiffIDs: []layer.DiffID{layer.DiffID(digest)}, }, } js, err := imageJSON.MarshalJSON() if err != nil { return -1, err } hash := sha256.New() hash.Write(js) layerID := hex.EncodeToString(hash.Sum(nil)) err = os.MkdirAll(s.options.HostPath("scratch", layerID), 0755) if err != nil { return -1, err } layerFile.Close() err = os.Rename(layerFile.Name(), s.options.HostPath("scratch", layerID, "layer.tar")) if err != nil { return -1, err } defer os.RemoveAll(s.options.HostPath("scratch")) // VERSION file versionFile, err := os.OpenFile(s.options.HostPath("scratch", layerID, "VERSION"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer versionFile.Close() _, err = versionFile.Write([]byte("1.0")) if err != nil { return -1, err } err = versionFile.Sync() if err != nil { return -1, err } // json file jsonFile, err := os.OpenFile(s.options.HostPath("scratch", layerID, "json"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer jsonFile.Close() _, err = jsonFile.Write(js) if err != nil { return -1, err } err = jsonFile.Sync() if err != nil { return -1, err } // repositories file repositoriesFile, err := os.OpenFile(s.options.HostPath("scratch", "repositories"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return -1, err } defer repositoriesFile.Close() _, err = repositoriesFile.Write([]byte(fmt.Sprintf(`{"%s":{`, s.authenticator.Repository(s.repository)))) if err != nil { return -1, err } if len(s.tags) == 0 { s.tags = []string{"latest"} } for i, tag := range s.tags { _, err = repositoriesFile.Write([]byte(fmt.Sprintf(`"%s":"%s"`, tag, layerID))) if err != nil { return -1, err } if i != len(s.tags)-1 { _, err = repositoriesFile.Write([]byte{','}) if err != nil { return -1, err } } } _, err = repositoriesFile.Write([]byte{'}', '}'}) err = repositoriesFile.Sync() if err != nil { return -1, err } // Build our output tarball and start writing to it imageFile, err := os.Create(s.options.HostPath("scratch.tar")) if err != nil { return -1, err } defer imageFile.Close() err = util.TarPath(imageFile, s.options.HostPath("scratch")) if err != nil { return -1, err } imageFile.Close() client, err := NewDockerClient(s.dockerOptions) if err != nil { return 1, err } // Check the auth if !s.dockerOptions.DockerLocal { check, err := s.authenticator.CheckAccess(s.repository, auth.Push) if !check || err != nil { s.logger.Errorln("Not allowed to interact with this repository:", s.repository) return -1, fmt.Errorf("Not allowed to interact with this repository: %s", s.repository) } } s.repository = s.authenticator.Repository(s.repository) s.logger.WithFields(util.LogFields{ "Repository": s.repository, "Tags": s.tags, "Message": s.message, }).Debug("Scratch push to registry") // Okay, we can access it, do a docker load to import the image then push it loadFile, err := os.Open(s.options.HostPath("scratch.tar")) if err != nil { return -1, err } defer loadFile.Close() e, err := core.EmitterFromContext(ctx) if err != nil { return 1, err } err = client.LoadImage(docker.LoadImageOptions{InputStream: loadFile}) if err != nil { return 1, err } return s.tagAndPush(layerID, e, client) }
// Execute runs a command and optionally reloads it func (s *WatchStep) Execute(ctx context.Context, sess *core.Session) (int, error) { e, err := core.EmitterFromContext(ctx) if err != nil { return -1, err } // TODO(termie): PACKAGING make this a feature of session and remove // the calls into its struct // Start watching our stdout stopListening := make(chan struct{}) defer func() { stopListening <- struct{}{} }() go func() { for { select { case line := <-sess.Recv(): e.Emit(core.Logs, &core.LogsArgs{ // Hidden: sess.logsHidden, Logs: line, }) // We need to make sure we stop eating the stdout from the container // promiscuously when we finish out step case <-stopListening: return } } }() // cheating to get containerID // TODO(termie): we should deal with this eventually dt := sess.Transport().(*DockerTransport) containerID := dt.containerID // Set up a signal handler to end our step. finishedStep := make(chan struct{}) stopWatchHandler := &util.SignalHandler{ ID: "stop-watch", // Signal our stuff to stop and finish the step, return false to // signify that we've handled the signal and don't process further F: func() bool { s.logger.Println("Keyboard interrupt detected, finishing step") finishedStep <- struct{}{} return false }, } util.GlobalSigint().Add(stopWatchHandler) // NOTE(termie): I think the only way to exit this code is via this // signal handler and the signal monkey removes handlers // after it processes them, so this may be superfluous defer util.GlobalSigint().Remove(stopWatchHandler) // If we're not going to reload just run the thing once, synchronously if !s.reload { err := sess.Send(ctx, false, "set +e", s.Code) if err != nil { return 0, err } <-finishedStep // ignoring errors s.killProcesses(containerID, "INT") return 0, nil } f := &util.Formatter{s.options.GlobalOptions.ShowColors} s.logger.Info(f.Info("Reloading on file changes")) doCmd := func() { err := sess.Send(ctx, false, "set +e", s.Code) if err != nil { s.logger.Errorln(err) return } open, err := exposedPortMaps(s.dockerOptions.DockerHost, s.options.PublishPorts) if err != nil { s.logger.Warnf(f.Info("There was a problem parsing your docker host."), err) return } for _, uri := range open { s.logger.Infof(f.Info("Forwarding %s to %s on the container."), uri.HostURI, uri.ContainerPort) } } // Otherwise set up a watcher and do some magic watcher, err := s.watch(s.options.ProjectPath) if err != nil { return -1, err } debounce := util.NewDebouncer(2 * time.Second) done := make(chan struct{}) go func() { for { select { case event := <-watcher.Events: s.logger.Debugln("fsnotify event", event.String()) if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Remove == fsnotify.Remove { if !strings.HasPrefix(filepath.Base(event.Name), ".") { s.logger.Debug(f.Info("Modified file", event.Name)) debounce.Trigger() } } case <-debounce.C: err := s.killProcesses(containerID, "INT") if err != nil { s.logger.Panic(err) return } s.logger.Info(f.Info("Reloading")) go doCmd() case err := <-watcher.Errors: s.logger.Error(err) done <- struct{}{} return case <-finishedStep: s.killProcesses(containerID, "INT") done <- struct{}{} return } } }() // Run build on first run debounce.Trigger() <-done return 0, nil }