func main() {
flag.Parse()
args := flag.Args()
if len(args) < 3 {
log.Printf("Need at least 3 args: <fileToMap:mapScript>... <reduceScript> <outDir>")
}
mapTasks := args[:len(args)-2]
reduceCmd := args[len(args)-2]
outDir := args[len(args)-1]
fmt.Printf("Args: %+v\n", args)
fmt.Printf("maps: %+v\n", mapTasks)
fmt.Printf("reduce script %s\n", reduceCmd)
fmt.Printf("outdir %s\n", outDir)
sshCfg := dsh.SshConf(*USER, *KEYFILE)
// dispatch map tasks piped through shuffle
mapCmds := make([]dsh.HostCmd, 0, 0)
for _, taskStr := range mapTasks {
// todo nicer error reporting on bad format
taskSplits := strings.Split(taskStr, ":")
glob := taskSplits[0]
cmd := taskSplits[1]
taskPaths, err := filepath.Glob(glob)
if err != nil {
panic(err)
}
splits, err := dsh.GetSplits(taskPaths)
if err != nil {
panic(err)
}
mapCmds = append(mapCmds, mapCommands(splits, cmd, *NUMREDUCE, outDir)...)
}
err := dsh.ExecShells(sshCfg, mapCmds, os.Stdout, os.Stderr)
if err != nil {
panic(err)
}
mapOutputs, err := filepath.Glob(fmt.Sprintf("%s/.mapOut*", outDir))
if err != nil {
panic(err)
}
defer func() {
for _, mapOut := range mapOutputs {
os.Remove(mapOut)
}
}()
// for each reduce task
reduceCmds, err := reduceCommands(outDir, *NUMREDUCE, reduceCmd)
if err != nil {
panic(err)
}
err = dsh.ExecShells(sshCfg, reduceCmds, os.Stdout, os.Stderr)
if err != nil {
panic(err)
}
}
// args are a list of filenames/globs (or directories) followed by a command
func main() {
flag.Parse()
args := flag.Args()
if len(args) < 2 {
log.Printf("Expected at least 2 args, <path..> <cmd>")
os.Exit(1)
}
var globs []string
if len(args) == 2 {
globs = []string{args[0]}
} else {
globs = args[:len(args)-1]
}
files := make([]string, 0)
for _, g := range globs {
matches, err := filepath.Glob(g)
if err != nil {
log.Printf("Error globbing path %s : %s", g, err)
}
fmt.Printf("Got matches %+v for glob %s\n", matches, g)
files = append(files, matches...)
}
splits, err := dsh.GetSplits(files)
if err != nil {
log.Printf("Error calculation splits for files %+v : %s", files, err)
os.Exit(1)
}
fmt.Printf("Splits: %+v\n", splits)
cmd := args[len(args)-1]
splitCmds := dsh.Commands(splits, cmd)
sshConfig := dsh.SshConf(*USER, *KEYFILE)
if err != nil {
log.Printf("Error getting ssh config: %s", err)
os.Exit(1)
}
err = dsh.ExecShells(sshConfig, splitCmds, os.Stdout, os.Stderr)
if err != nil {
log.Printf("Error execing commands %s", err)
os.Exit(1)
}
}