// input makes a dataset available for computations in the container. func (p *pipeline) input(name string) error { var trimmed string switch { case strings.HasPrefix(name, "s3://"): trimmed = strings.TrimPrefix(name, "s3://") if err := WaitPipeline(p.pipelineDir, trimmed, p.commit); err != nil { return err } hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit)) containerPath := path.Join("/in", trimmed) bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath) p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) case strings.HasPrefix(name, "pps://"): trimmed = strings.TrimPrefix(name, "pps://") err := WaitPipeline(p.pipelineDir, trimmed, p.commit) if err != nil { return err } hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit)) containerPath := path.Join("/in", trimmed) bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath) p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) case strings.HasPrefix(name, "pfs://"): fallthrough default: hostPath := btrfs.HostPath(path.Join(p.inRepo, p.commit, name)) containerPath := path.Join("/in", name) bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath) p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) } return nil }
func (p *pipeline) bind(repo string, directory string, containerPath string) error { hostPath := btrfs.HostPath(path.Join(repo, p.commit, directory)) //p.config.Config.Volumes[containerPath] = emptyStruct() bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath) p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) if err := btrfs.Show(repo, p.commit, p.commit+"-new"); err != nil { return err } p.createdCommits = append(p.createdCommits, path.Join(repo, p.commit+"-new")) hostPath = btrfs.HostPath(path.Join(repo, p.commit+"-new", directory)) //p.config.Config.Volumes[containerPath+"-new"] = emptyStruct() bind = fmt.Sprintf("%s:%s:ro", hostPath, containerPath+"-new") p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) return nil }
// Run runs a command in the container, it assumes that `branch` has already // been created. // Notice that any failure in this function leads to the branch having // uncommitted dirty changes. This state needs to be cleaned up before the // pipeline is rerun. The reason we don't do it here is that even if we try our // best the process crashing at the wrong time could still leave it in an // inconsistent state. func (p *pipeline) run(cmd []string) error { // this function always increments counter defer func() { p.counter++ }() // Check if the commit already exists exists, err := btrfs.FileExists(path.Join(p.outRepo, p.runCommit())) if err != nil { return err } // if the commit exists there's no work to be done if exists { return nil } // Set the command p.config.Config.Cmd = []string{"sh"} //p.config.Config.Volumes["/out"] = emptyStruct() // Map the out directory in as a bind hostPath := btrfs.HostPath(path.Join(p.outRepo, p.branch)) bind := fmt.Sprintf("%s:/out", hostPath) p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind) log.Print(p.config.HostConfig.Binds) // Make sure this bind is only visible for the duration of run defer func() { p.config.HostConfig.Binds = p.config.HostConfig.Binds[:len(p.config.HostConfig.Binds)-1] }() // Start the container p.container, err = startContainer(p.config) if err != nil { return err } if err := pipeToStdin(p.container, strings.NewReader(strings.Join(cmd, " ")+"\n")); err != nil { return err } // Create a place to put the logs f, err := btrfs.CreateAll(path.Join(p.outRepo, p.branch, ".log")) if err != nil { return err } defer f.Close() // Copy the logs from the container in to the file. if err = containerLogs(p.container, f); err != nil { return err } // Wait for the command to finish: exit, err := waitContainer(p.container) if err != nil { return err } if exit != 0 { // The command errored return fmt.Errorf("Command:\n\t%s\nhad exit code: %d.\n", strings.Join(cmd, " "), exit) } return btrfs.Commit(p.outRepo, p.runCommit(), p.branch) }