func callCont(c *exec.Cmd, cont func(io.Reader) error) error {
_, callerFile, callerLine, _ := runtime.Caller(1)
log.Printf("%15s:%.3d -> %s", path.Base(callerFile), callerLine, strings.Join(c.Args, " "))
var reader io.Reader
var err error
if cont != nil {
reader, err = c.StdoutPipe()
if err != nil {
return err
}
}
stderr, err := c.StderrPipe()
if err != nil {
return err
}
if err = c.Start(); err != nil {
return err
}
if cont != nil {
if err = cont(reader); err != nil {
return err
}
}
buffer := bytes.NewBuffer(nil)
buffer.ReadFrom(stderr)
if buffer.Len() != 0 {
log.Print("Command had output on stderr.\n Cmd: ", strings.Join(c.Args, " "), "\nstderr: ", buffer)
}
return c.Wait()
}
func (r *runner) Start(pipelineSource *pps.PipelineSource) (string, error) {
dirPath, pipeline, err := r.sourcer.GetDirPathAndPipeline(pipelineSource)
if err != nil {
return "", err
}
pipelineInfo, err := r.grapher.GetPipelineInfo(pipeline)
if err != nil {
return "", err
}
log.Printf("%v %v %v\n", dirPath, pipeline, pipelineInfo)
return "1234", nil
}
// bestRole returns the best role for us to fill in the cluster right now. If
// no shards are available it returns ErrNoShards.
func (s *shard) freeRole() (uint64, error) {
masters, err := s.masters()
log.Printf("%#v", masters)
if err != nil {
return 0, err
}
// First we check if there's an empty shard
for i, master := range masters {
if master == "" {
return uint64(i), nil
}
}
return 0, ErrNoShards
}
func parsePipelineV1(dirPath string, contextDirPath string, config *config) (*pps.Pipeline, error) {
filePaths, err := getAllFilePaths(dirPath, contextDirPath, config.Include, config.Exclude)
if err != nil {
return nil, err
}
pipeline := &pps.Pipeline{
NameToElement: make(map[string]*pps.Element),
}
for _, filePath := range filePaths {
element, err := getElementForPipelineFile(dirPath, filePath)
if err != nil {
return nil, err
}
if _, ok := pipeline.NameToElement[element.Name]; ok {
return nil, fmt.Errorf("duplicate element: %s", element.Name)
}
pipeline.NameToElement[element.Name] = element
}
log.Printf("got pipeline %v\n", pipeline)
return pipeline, nil
}
func getPipelineInfo(pipeline *pps.Pipeline) (*PipelineInfo, error) {
nodes := getNameToNode(pipeline)
nodeToInputs := getNodeNameToInputStrings(nodes)
nodeToOutputs := getNodeNameToOutputStrings(nodes)
inputToNodes := getInputStringToNodeNames(nodes)
outputToNodes := getOutputStringToNodeNames(nodes)
nodeInfos := make(map[string](*NodeInfo))
for name := range nodes {
nodeInfo := &NodeInfo{
Parents: make([]string, 0),
Children: make([]string, 0),
}
parents := make(map[string]bool)
for input := range nodeToInputs[name] {
for parent := range outputToNodes[input] {
if parent != name {
parents[parent] = true
}
}
}
for parent := range parents {
nodeInfo.Parents = append(nodeInfo.Parents, parent)
}
children := make(map[string]bool)
for output := range nodeToOutputs[name] {
for child := range inputToNodes[output] {
if child != name {
children[child] = true
}
}
}
for child := range children {
nodeInfo.Children = append(nodeInfo.Children, child)
}
nodeInfos[name] = nodeInfo
}
pipelineInfo := &PipelineInfo{nodeInfos}
log.Printf("got pipeline info %v\n", pipelineInfo)
return pipelineInfo, nil
}
