Example #1
0
File: main.go Project: jbooth/dsh
func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 3 {
		log.Printf("Need at least 3 args:  <fileToMap:mapScript>... <reduceScript> <outDir>")
	}
	mapTasks := args[:len(args)-2]
	reduceCmd := args[len(args)-2]
	outDir := args[len(args)-1]

	fmt.Printf("Args: %+v\n", args)
	fmt.Printf("maps: %+v\n", mapTasks)
	fmt.Printf("reduce script %s\n", reduceCmd)
	fmt.Printf("outdir %s\n", outDir)
	sshCfg := dsh.SshConf(*USER, *KEYFILE)
	// dispatch map tasks piped through shuffle
	mapCmds := make([]dsh.HostCmd, 0, 0)
	for _, taskStr := range mapTasks {
		// todo nicer error reporting on bad format
		taskSplits := strings.Split(taskStr, ":")
		glob := taskSplits[0]
		cmd := taskSplits[1]
		taskPaths, err := filepath.Glob(glob)
		if err != nil {
			panic(err)
		}
		splits, err := dsh.GetSplits(taskPaths)
		if err != nil {
			panic(err)
		}
		mapCmds = append(mapCmds, mapCommands(splits, cmd, *NUMREDUCE, outDir)...)
	}
	err := dsh.ExecShells(sshCfg, mapCmds, os.Stdout, os.Stderr)
	if err != nil {
		panic(err)
	}
	mapOutputs, err := filepath.Glob(fmt.Sprintf("%s/.mapOut*", outDir))
	if err != nil {
		panic(err)
	}
	defer func() {
		for _, mapOut := range mapOutputs {
			os.Remove(mapOut)
		}
	}()
	// for each reduce task
	reduceCmds, err := reduceCommands(outDir, *NUMREDUCE, reduceCmd)
	if err != nil {
		panic(err)
	}
	err = dsh.ExecShells(sshCfg, reduceCmds, os.Stdout, os.Stderr)
	if err != nil {
		panic(err)
	}
}
Example #2
0
File: main.go Project: jbooth/dsh
// args are a list of filenames/globs (or directories) followed by a command
func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 2 {
		log.Printf("Expected at least 2 args, <path..> <cmd>")
		os.Exit(1)
	}
	var globs []string
	if len(args) == 2 {
		globs = []string{args[0]}
	} else {
		globs = args[:len(args)-1]
	}
	files := make([]string, 0)
	for _, g := range globs {
		matches, err := filepath.Glob(g)
		if err != nil {
			log.Printf("Error globbing path %s : %s", g, err)
		}
		fmt.Printf("Got matches %+v for glob %s\n", matches, g)
		files = append(files, matches...)
	}
	splits, err := dsh.GetSplits(files)
	if err != nil {
		log.Printf("Error calculation splits for files %+v : %s", files, err)
		os.Exit(1)
	}
	fmt.Printf("Splits: %+v\n", splits)
	cmd := args[len(args)-1]
	splitCmds := dsh.Commands(splits, cmd)
	sshConfig := dsh.SshConf(*USER, *KEYFILE)
	if err != nil {
		log.Printf("Error getting ssh config: %s", err)
		os.Exit(1)
	}
	err = dsh.ExecShells(sshConfig, splitCmds, os.Stdout, os.Stderr)
	if err != nil {
		log.Printf("Error execing commands %s", err)
		os.Exit(1)
	}
}