func worker() error {
// Change current directory to that of submission.
wd, err := getenv("PBS_O_WORKDIR")
if err != nil {
return err
}
if err := os.Chdir(wd); err != nil {
return fmt.Errorf("chdir: %v", err)
}
// Determine file locations.
var inFile, outFile, errFile string
if workerMapLen > 0 {
// If this is a map task, then use the array index.
var ind int
// Array index cannot be set for maps of 1 job.
// In this case the index is zero.
if workerMapLen > 1 {
var err error
ind, err = getenvInt("PBS_ARRAY_INDEX")
if err != nil {
return err
}
// Convert to zero-indexed.
ind--
}
inFile = fmt.Sprintf("in-%d.json", ind)
outFile = fmt.Sprintf("out-%d.json", ind)
errFile = fmt.Sprintf("err-%d.json", ind)
} else {
inFile = "in.json"
outFile = "out.json"
errFile = "err.json"
}
inFile = path.Join(workerDir, inFile)
outFile = path.Join(workerDir, outFile)
errFile = path.Join(workerDir, errFile)
// Config file does not vary with index.
confFile := path.Join(workerDir, "conf.json")
// Error can only be communicated once the task ID has been determined.
if err := doTask(inFile, confFile, outFile); err != nil {
// Attempt to save error.
if err := fileutil.SaveExt(errFile, err.Error()); err != nil {
return fmt.Errorf("save error: %v", err)
}
}
return nil
}
// An error returned by this function will be communicated to the master.
// Or at least we will try.
// This can only be done once the task ID has been determined.
func doTask(inFile, confFile, outFile string) error {
// Look up task by name.
var task ConfigTask
if workerMapLen > 0 {
spec, there := mapTasks[workerTask]
if !there {
return fmt.Errorf(`map task not found: "%s"`, workerTask)
}
task = spec.Task
} else {
spec, there := tasks[workerTask]
if !there {
return fmt.Errorf(`task not found: "%s"`, workerTask)
}
task = spec.Task
}
x := task.NewInput()
if x != nil {
log.Println("load input:", inFile)
if err := fileutil.LoadExt(inFile, x); err != nil {
return fmt.Errorf("load input: %v", err)
}
x = deref(x)
}
p := task.NewConfig()
if p != nil {
log.Println("load config:", confFile)
if err := fileutil.LoadExt(confFile, p); err != nil {
return fmt.Errorf("load config: %v", err)
}
p = deref(p)
}
log.Println("call function")
y, err := task.Func(x, p)
if err != nil {
return err
}
if y != nil {
log.Println("save output:", outFile)
if err := fileutil.SaveExt(outFile, y); err != nil {
return fmt.Errorf("save output: %v", err)
}
}
return nil
}
// Map computes y[i] = f(x[i], p) for all i.
//
// The function is specified by name and must already be registered.
// The input x must be a slice or similar (see reflect.Value.Index()).
// The output y must be a pointer to a slice.
// If the length of y is sufficient to hold the output, it will be over-written.
// If it is not sufficient, a new array will be allocated.
// After a succesful call, the length of y will match that of x.
func Map(f string, y, x, p interface{}, stdout, stderr io.Writer, flags []string) error {
task, there := mapTasks[f]
if !there {
return fmt.Errorf(`map task not found: "%s"`, f)
}
// Recursively invoked closure.
var do func(task *mapTaskSpec, y, x interface{}, chunk bool) (string, error)
do = func(task *mapTaskSpec, y, x interface{}, chunk bool) (string, error) {
n := reflect.ValueOf(x).Len()
y = ensureLenAndDeref(y, n)
// y now has correct len, is not a pointer, and can be modified.
if chunk {
u, inds := split(x, 1, max(task.ChunkLen, 1))
// Create slice of slices for output.
vtyp := reflect.SliceOf(reflect.TypeOf(y))
v := reflect.New(vtyp).Interface()
dir, err := do(task, v, u, false)
v = deref(v)
if err != nil {
mapErr := err.(MapError)
// Need to re-map task errors.
taskErrs := make(map[int]error)
for i := range inds {
if err := mapErr.Tasks[i]; err != nil {
// Give error to all members.
for _, p := range inds[i] {
taskErrs[p] = err
}
continue
}
// No error occured. Move outputs.
for j, p := range inds[i] {
vij := reflect.ValueOf(v).Index(i).Index(j)
yp := reflect.ValueOf(y).Index(p)
yp.Set(vij)
}
}
return dir, MapError{mapErr.Master, taskErrs, n}
}
mergeTo(y, v)
return dir, nil
}
// Create temporary directory.
wd, err := os.Getwd()
if err != nil {
return "", err
}
dir, err := ioutil.TempDir(wd, f+"-")
if err != nil {
return "", err
}
// Save each input to file.
xval := reflect.ValueOf(x)
for i := 0; i < xval.Len(); i++ {
inFile := path.Join(dir, fmt.Sprintf("in-%d.json", i))
err := fileutil.SaveExt(inFile, xval.Index(i).Interface())
if err != nil {
return dir, fmt.Errorf("save input %d: %v", i, err)
}
}
if p != nil {
confFile := path.Join(dir, "conf.json")
err := fileutil.SaveExt(confFile, p)
if err != nil {
return dir, fmt.Errorf("save config: %v", err)
}
}
// Invoke qsub.
jobargs := []string{"-dstrfn.task", f, "-dstrfn.map", fmt.Sprint(n), "-dstrfn.dir", dir}
if len(flags) > 0 {
jobargs = append(jobargs, flags...)
}
execErr, err := submit(n, jobargs, f, dir, task.Flags, nil, nil)
if err != nil {
return dir, err
}
taskErrs := make(map[int]error)
for i := 0; i < n; i++ {
// Load from output file.
outFile := path.Join(dir, fmt.Sprintf("out-%d.json", i))
errFile := path.Join(dir, fmt.Sprintf("err-%d.json", i))
yi := reflect.ValueOf(y).Index(i).Addr().Interface()
if _, err := os.Stat(outFile); err == nil {
// If output file exists, attempt to load.
if err := fileutil.LoadExt(outFile, yi); err != nil {
taskErrs[i] = fmt.Errorf("load output: %v", err)
continue
}
} else if !os.IsNotExist(err) {
// Could not stat file.
taskErrs[i] = err
continue
} else {
// Output file did not exist. Try to load error file.
if _, err := os.Stat(errFile); err == nil {
// Error file exists. Attempt to load.
var str string
if err := fileutil.LoadExt(errFile, &str); err != nil {
taskErrs[i] = err
continue
}
taskErrs[i] = errors.New(str)
continue
} else if !os.IsNotExist(err) {
// Could not stat file.
taskErrs[i] = err
continue
}
taskErrs[i] = fmt.Errorf("could not find output or error files: job %d", i)
continue
}
}
if execErr != nil {
return dir, MapError{execErr, taskErrs, n}
}
if len(taskErrs) > 0 {
return dir, MapError{Tasks: taskErrs, Len: n}
}
return dir, nil
}
tmpdir, err := do(task, y, x, task.Chunk)
if err != nil {
return err
}
// Only remove temporary directory if there was no error.
if !debug {
if err := removeAll(tmpdir); err != nil {
log.Println(err)
}
}
return nil
}
