// An error returned by this function will be communicated to the master.
// Or at least we will try.
// This can only be done once the task ID has been determined.
func doTask(inFile, confFile, outFile string) error {
// Look up task by name.
var task ConfigTask
if workerMapLen > 0 {
spec, there := mapTasks[workerTask]
if !there {
return fmt.Errorf(`map task not found: "%s"`, workerTask)
}
task = spec.Task
} else {
spec, there := tasks[workerTask]
if !there {
return fmt.Errorf(`task not found: "%s"`, workerTask)
}
task = spec.Task
}
x := task.NewInput()
if x != nil {
log.Println("load input:", inFile)
if err := fileutil.LoadExt(inFile, x); err != nil {
return fmt.Errorf("load input: %v", err)
}
x = deref(x)
}
p := task.NewConfig()
if p != nil {
log.Println("load config:", confFile)
if err := fileutil.LoadExt(confFile, p); err != nil {
return fmt.Errorf("load config: %v", err)
}
p = deref(p)
}
log.Println("call function")
y, err := task.Func(x, p)
if err != nil {
return err
}
if y != nil {
log.Println("save output:", outFile)
if err := fileutil.SaveExt(outFile, y); err != nil {
return fmt.Errorf("save output: %v", err)
}
}
return nil
}
// Call calls the function and saves the output to the specified file.
// It does not load the result into memory.
// If the file already exists, it does not call the function.
func Call(f string, y, x interface{}, stdout, stderr io.Writer, flags []string) error {
task, there := tasks[f]
if !there {
return fmt.Errorf(`task not found: "%s"`, f)
}
// Create temporary directory.
dir, err := ioutil.TempDir(".", f+"-")
if err != nil {
return err
}
inFile := path.Join(dir, "in.json")
outFile := path.Join(dir, "out.json")
errFile := path.Join(dir, "err.json")
// Save input.
if err := fileutil.SaveJSON(inFile, x); err != nil {
return err
}
// Invoke qsub.
jobargs := []string{"-dstrfn.task", f, "-dstrfn.dir", dir}
if len(flags) > 0 {
jobargs = append(jobargs, flags...)
}
execErr, err := submit(1, jobargs, f, dir, task.Flags, stdout, stderr)
if err != nil {
return err
}
if execErr != nil {
return execErr
}
if _, err := os.Stat(errFile); err == nil {
// Error file exists. Attempt to load.
var str string
if err := fileutil.LoadExt(errFile, &str); err != nil {
return fmt.Errorf("load error file: %v", err)
}
return errors.New(str)
} else if !os.IsNotExist(err) {
// Could not stat file.
return fmt.Errorf("stat error file: %v", err)
}
// Error file does not exist.
if y != nil {
// Output required.
if _, err := os.Stat(outFile); os.IsNotExist(err) {
return errors.New("could not find output or error files")
} else if err != nil {
return fmt.Errorf("stat output file: %v", err)
}
if err := fileutil.LoadExt(outFile, y); err != nil {
return err
}
}
// Only remove temporary directory if there was no error.
if !debug {
return removeAll(dir)
}
return nil
}
func main() {
var (
// Dataset options.
name = flag.String("dataset", "usatest", "Dataset identifier.")
dir = flag.String("dir", "", "Location of dataset. Empty means working dir.")
// Detection options.
pyrStep = flag.Float64("pyr-step", 1.2, "Geometric scale steps in image pyramid.")
maxScale = flag.Float64("max-scale", 1, "Maximum amount to scale image. Greater than 1 is upsampling.")
maxIOU = flag.Float64("max-iou", 0.3, "Maximum IOU that two detections can have before NMS.")
margin = flag.Int("margin", 0, "Spatial bin parameter to HOG.")
// Validation options.
minValIOU = flag.Float64("min-val-iou", 0.5, "Minimum IOU for a detection to be validated.")
minIgnoreCover = flag.Float64("min-ignore-cover", 0, "Minimum that a detection must be covered by an ignore region to be ignored.")
// Display options.
numShow = flag.Int("num-show", 4, "Number of detections to show per image")
)
flag.Parse()
if flag.NArg() != 2 {
flag.Usage()
os.Exit(1)
}
var (
tmplFile = flag.Arg(0)
transformFile = flag.Arg(1)
)
// Get dataset from name.
dataset, err := datasetByName(*name)
if err != nil {
log.Fatalln(err)
}
// Load template from file.
var tmpl *detect.FeatTmpl
if err := fileutil.LoadExt(tmplFile, &tmpl); err != nil {
log.Fatalln("load template:", err)
}
// Load transform from file.
var transform *feat.Marshaler
if err := fileutil.LoadJSON(transformFile, &transform); err != nil {
log.Fatalln("load transform:", err)
}
overlap := func(a, b image.Rectangle) bool { return detect.IOU(a, b) > *maxIOU }
opts := detect.MultiScaleOpts{
MaxScale: *maxScale,
PyrStep: *pyrStep,
Interp: resize.Bicubic,
Transform: transform,
Pad: feat.Pad{feat.UniformMargin(*margin), imsamp.Continue},
DetFilter: detect.DetFilter{LocalMax: true, MinScore: math.Inf(-1)},
SupprFilter: detect.SupprFilter{MaxNum: 0, Overlap: overlap},
}
var val *detect.ValSet
err = fileutil.Cache(&val, "val-set.json", func() (*detect.ValSet, error) {
return testAll(dataset, *dir, tmpl, opts, *minValIOU, *minIgnoreCover, *numShow)
})
if err != nil {
log.Fatal(err)
}
}
// Map computes y[i] = f(x[i], p) for all i.
//
// The function is specified by name and must already be registered.
// The input x must be a slice or similar (see reflect.Value.Index()).
// The output y must be a pointer to a slice.
// If the length of y is sufficient to hold the output, it will be over-written.
// If it is not sufficient, a new array will be allocated.
// After a succesful call, the length of y will match that of x.
func Map(f string, y, x, p interface{}, stdout, stderr io.Writer, flags []string) error {
task, there := mapTasks[f]
if !there {
return fmt.Errorf(`map task not found: "%s"`, f)
}
// Recursively invoked closure.
var do func(task *mapTaskSpec, y, x interface{}, chunk bool) (string, error)
do = func(task *mapTaskSpec, y, x interface{}, chunk bool) (string, error) {
n := reflect.ValueOf(x).Len()
y = ensureLenAndDeref(y, n)
// y now has correct len, is not a pointer, and can be modified.
if chunk {
u, inds := split(x, 1, max(task.ChunkLen, 1))
// Create slice of slices for output.
vtyp := reflect.SliceOf(reflect.TypeOf(y))
v := reflect.New(vtyp).Interface()
dir, err := do(task, v, u, false)
v = deref(v)
if err != nil {
mapErr := err.(MapError)
// Need to re-map task errors.
taskErrs := make(map[int]error)
for i := range inds {
if err := mapErr.Tasks[i]; err != nil {
// Give error to all members.
for _, p := range inds[i] {
taskErrs[p] = err
}
continue
}
// No error occured. Move outputs.
for j, p := range inds[i] {
vij := reflect.ValueOf(v).Index(i).Index(j)
yp := reflect.ValueOf(y).Index(p)
yp.Set(vij)
}
}
return dir, MapError{mapErr.Master, taskErrs, n}
}
mergeTo(y, v)
return dir, nil
}
// Create temporary directory.
wd, err := os.Getwd()
if err != nil {
return "", err
}
dir, err := ioutil.TempDir(wd, f+"-")
if err != nil {
return "", err
}
// Save each input to file.
xval := reflect.ValueOf(x)
for i := 0; i < xval.Len(); i++ {
inFile := path.Join(dir, fmt.Sprintf("in-%d.json", i))
err := fileutil.SaveExt(inFile, xval.Index(i).Interface())
if err != nil {
return dir, fmt.Errorf("save input %d: %v", i, err)
}
}
if p != nil {
confFile := path.Join(dir, "conf.json")
err := fileutil.SaveExt(confFile, p)
if err != nil {
return dir, fmt.Errorf("save config: %v", err)
}
}
// Invoke qsub.
jobargs := []string{"-dstrfn.task", f, "-dstrfn.map", fmt.Sprint(n), "-dstrfn.dir", dir}
if len(flags) > 0 {
jobargs = append(jobargs, flags...)
}
execErr, err := submit(n, jobargs, f, dir, task.Flags, nil, nil)
if err != nil {
return dir, err
}
taskErrs := make(map[int]error)
for i := 0; i < n; i++ {
// Load from output file.
outFile := path.Join(dir, fmt.Sprintf("out-%d.json", i))
errFile := path.Join(dir, fmt.Sprintf("err-%d.json", i))
yi := reflect.ValueOf(y).Index(i).Addr().Interface()
if _, err := os.Stat(outFile); err == nil {
// If output file exists, attempt to load.
if err := fileutil.LoadExt(outFile, yi); err != nil {
taskErrs[i] = fmt.Errorf("load output: %v", err)
continue
}
} else if !os.IsNotExist(err) {
// Could not stat file.
taskErrs[i] = err
continue
} else {
// Output file did not exist. Try to load error file.
if _, err := os.Stat(errFile); err == nil {
// Error file exists. Attempt to load.
var str string
if err := fileutil.LoadExt(errFile, &str); err != nil {
taskErrs[i] = err
continue
}
taskErrs[i] = errors.New(str)
continue
} else if !os.IsNotExist(err) {
// Could not stat file.
taskErrs[i] = err
continue
}
taskErrs[i] = fmt.Errorf("could not find output or error files: job %d", i)
continue
}
}
if execErr != nil {
return dir, MapError{execErr, taskErrs, n}
}
if len(taskErrs) > 0 {
return dir, MapError{Tasks: taskErrs, Len: n}
}
return dir, nil
}
tmpdir, err := do(task, y, x, task.Chunk)
if err != nil {
return err
}
// Only remove temporary directory if there was no error.
if !debug {
if err := removeAll(tmpdir); err != nil {
log.Println(err)
}
}
return nil
}