// input makes a dataset available for computations in the container.
func (p *pipeline) input(name string) error {
var trimmed string
switch {
case strings.HasPrefix(name, "s3://"):
trimmed = strings.TrimPrefix(name, "s3://")
if err := WaitPipeline(p.pipelineDir, trimmed, p.commit); err != nil {
return err
}
hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit))
containerPath := path.Join("/in", trimmed)
bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
case strings.HasPrefix(name, "pps://"):
trimmed = strings.TrimPrefix(name, "pps://")
err := WaitPipeline(p.pipelineDir, trimmed, p.commit)
if err != nil {
return err
}
hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit))
containerPath := path.Join("/in", trimmed)
bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
case strings.HasPrefix(name, "pfs://"):
fallthrough
default:
hostPath := btrfs.HostPath(path.Join(p.inRepo, p.commit, name))
containerPath := path.Join("/in", name)
bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
}
return nil
}
func (p *pipeline) bind(repo string, directory string, containerPath string) error {
hostPath := btrfs.HostPath(path.Join(repo, p.commit, directory))
//p.config.Config.Volumes[containerPath] = emptyStruct()
bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
if err := btrfs.Show(repo, p.commit, p.commit+"-new"); err != nil {
return err
}
p.createdCommits = append(p.createdCommits, path.Join(repo, p.commit+"-new"))
hostPath = btrfs.HostPath(path.Join(repo, p.commit+"-new", directory))
//p.config.Config.Volumes[containerPath+"-new"] = emptyStruct()
bind = fmt.Sprintf("%s:%s:ro", hostPath, containerPath+"-new")
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
return nil
}
// Run runs a command in the container, it assumes that `branch` has already
// been created.
// Notice that any failure in this function leads to the branch having
// uncommitted dirty changes. This state needs to be cleaned up before the
// pipeline is rerun. The reason we don't do it here is that even if we try our
// best the process crashing at the wrong time could still leave it in an
// inconsistent state.
func (p *pipeline) run(cmd []string) error {
// this function always increments counter
defer func() { p.counter++ }()
// Check if the commit already exists
exists, err := btrfs.FileExists(path.Join(p.outRepo, p.runCommit()))
if err != nil {
return err
}
// if the commit exists there's no work to be done
if exists {
return nil
}
// Set the command
p.config.Config.Cmd = []string{"sh"}
//p.config.Config.Volumes["/out"] = emptyStruct()
// Map the out directory in as a bind
hostPath := btrfs.HostPath(path.Join(p.outRepo, p.branch))
bind := fmt.Sprintf("%s:/out", hostPath)
p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
log.Print(p.config.HostConfig.Binds)
// Make sure this bind is only visible for the duration of run
defer func() { p.config.HostConfig.Binds = p.config.HostConfig.Binds[:len(p.config.HostConfig.Binds)-1] }()
// Start the container
p.container, err = startContainer(p.config)
if err != nil {
return err
}
if err := pipeToStdin(p.container, strings.NewReader(strings.Join(cmd, " ")+"\n")); err != nil {
return err
}
// Create a place to put the logs
f, err := btrfs.CreateAll(path.Join(p.outRepo, p.branch, ".log"))
if err != nil {
return err
}
defer f.Close()
// Copy the logs from the container in to the file.
if err = containerLogs(p.container, f); err != nil {
return err
}
// Wait for the command to finish:
exit, err := waitContainer(p.container)
if err != nil {
return err
}
if exit != 0 {
// The command errored
return fmt.Errorf("Command:\n\t%s\nhad exit code: %d.\n",
strings.Join(cmd, " "), exit)
}
return btrfs.Commit(p.outRepo, p.runCommit(), p.branch)
}