Esempio n. 1
0
// input makes a dataset available for computations in the container.
func (p *pipeline) input(name string) error {
	var trimmed string
	switch {
	case strings.HasPrefix(name, "s3://"):
		trimmed = strings.TrimPrefix(name, "s3://")
		if err := WaitPipeline(p.pipelineDir, trimmed, p.commit); err != nil {
			return err
		}
		hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit))
		containerPath := path.Join("/in", trimmed)
		bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
		p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	case strings.HasPrefix(name, "pps://"):
		trimmed = strings.TrimPrefix(name, "pps://")
		err := WaitPipeline(p.pipelineDir, trimmed, p.commit)
		if err != nil {
			return err
		}
		hostPath := btrfs.HostPath(path.Join(p.pipelineDir, trimmed, p.commit))
		containerPath := path.Join("/in", trimmed)
		bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
		p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	case strings.HasPrefix(name, "pfs://"):
		fallthrough
	default:
		hostPath := btrfs.HostPath(path.Join(p.inRepo, p.commit, name))
		containerPath := path.Join("/in", name)
		bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
		p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	}
	return nil
}
Esempio n. 2
0
func (p *pipeline) bind(repo string, directory string, containerPath string) error {
	hostPath := btrfs.HostPath(path.Join(repo, p.commit, directory))
	//p.config.Config.Volumes[containerPath] = emptyStruct()
	bind := fmt.Sprintf("%s:%s:ro", hostPath, containerPath)
	p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	if err := btrfs.Show(repo, p.commit, p.commit+"-new"); err != nil {
		return err
	}
	p.createdCommits = append(p.createdCommits, path.Join(repo, p.commit+"-new"))
	hostPath = btrfs.HostPath(path.Join(repo, p.commit+"-new", directory))
	//p.config.Config.Volumes[containerPath+"-new"] = emptyStruct()
	bind = fmt.Sprintf("%s:%s:ro", hostPath, containerPath+"-new")
	p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	return nil
}
Esempio n. 3
0
// Run runs a command in the container, it assumes that `branch` has already
// been created.
// Notice that any failure in this function leads to the branch having
// uncommitted dirty changes. This state needs to be cleaned up before the
// pipeline is rerun. The reason we don't do it here is that even if we try our
// best the process crashing at the wrong time could still leave it in an
// inconsistent state.
func (p *pipeline) run(cmd []string) error {
	// this function always increments counter
	defer func() { p.counter++ }()
	// Check if the commit already exists
	exists, err := btrfs.FileExists(path.Join(p.outRepo, p.runCommit()))
	if err != nil {
		return err
	}
	// if the commit exists there's no work to be done
	if exists {
		return nil
	}
	// Set the command
	p.config.Config.Cmd = []string{"sh"}
	//p.config.Config.Volumes["/out"] = emptyStruct()
	// Map the out directory in as a bind
	hostPath := btrfs.HostPath(path.Join(p.outRepo, p.branch))
	bind := fmt.Sprintf("%s:/out", hostPath)
	p.config.HostConfig.Binds = append(p.config.HostConfig.Binds, bind)
	log.Print(p.config.HostConfig.Binds)
	// Make sure this bind is only visible for the duration of run
	defer func() { p.config.HostConfig.Binds = p.config.HostConfig.Binds[:len(p.config.HostConfig.Binds)-1] }()
	// Start the container
	p.container, err = startContainer(p.config)
	if err != nil {
		return err
	}
	if err := pipeToStdin(p.container, strings.NewReader(strings.Join(cmd, " ")+"\n")); err != nil {
		return err
	}
	// Create a place to put the logs
	f, err := btrfs.CreateAll(path.Join(p.outRepo, p.branch, ".log"))
	if err != nil {
		return err
	}
	defer f.Close()
	// Copy the logs from the container in to the file.
	if err = containerLogs(p.container, f); err != nil {
		return err
	}
	// Wait for the command to finish:
	exit, err := waitContainer(p.container)
	if err != nil {
		return err
	}
	if exit != 0 {
		// The command errored
		return fmt.Errorf("Command:\n\t%s\nhad exit code: %d.\n",
			strings.Join(cmd, " "), exit)
	}
	return btrfs.Commit(p.outRepo, p.runCommit(), p.branch)
}