func main() { var master string var confFile string var inputs Inputs var worker string var jobtype string const ( defaultMaster = "localhost" masterUsage = "The master node." defaultConf = "/etc/disco/settings.py" confUsage = "The setting file which contains disco settings" defaultWorker = "" workerUsage = "The worker directory, a .go file, or an executable" defaultInputs = "" inputUsage = "The comma separated list of inputs to the job." defaultJobType = "mapreduce" jobTypeUsage = "type of the job (mapreduce or pipeline)" ) flag.StringVar(&master, "Master", "", masterUsage) flag.StringVar(&master, "M", "", masterUsage) flag.StringVar(&confFile, "Conf", defaultConf, confUsage) flag.StringVar(&confFile, "C", defaultConf, confUsage) flag.StringVar(&worker, "Worker", defaultWorker, workerUsage) flag.StringVar(&worker, "W", defaultWorker, workerUsage) flag.Var(&inputs, "Inputs", inputUsage) flag.Var(&inputs, "I", inputUsage) flag.StringVar(&jobtype, "Type", defaultJobType, jobTypeUsage) flag.StringVar(&jobtype, "T", defaultJobType, jobTypeUsage) flag.Parse() if worker == "" || len(inputs) == 0 { fmt.Println("Usage: jobpack -W worker_dir -I input(s)") os.Exit(1) } jobutil.AddFile(confFile) if master != "" { jobutil.SetKeyValue("DISCO_MASTER_HOST", master) } else if jobutil.Setting("DISCO_MASTER_HOST") == "" { jobutil.SetKeyValue("DISCO_MASTER_HOST", defaultMaster) } CreateJobPack(inputs, worker, jobtype) Post() Cleanup() }
func Run(Map Process, Reduce Process) { var w Worker send_worker() w.task = request_task() jobutil.SetKeyValue("HOST", w.task.Host) master, port := jobutil.HostAndPort(w.task.Master) jobutil.SetKeyValue("DISCO_MASTER_HOST", master) if port != fmt.Sprintf("%d", w.task.Disco_port) { panic("port mismatch: " + port) } jobutil.SetKeyValue("DISCO_PORT", port) jobutil.SetKeyValue("PUT_PORT", string(w.task.Put_port)) jobutil.SetKeyValue("DISCO_DATA", w.task.Disco_data) jobutil.SetKeyValue("DDFS_DATA", w.task.Ddfs_data) w.inputs = request_input() pwd, err := os.Getwd() Check(err) if w.task.Stage == "map" { w.runStage(pwd, "map_out_", Map) } else if w.task.Stage == "map_shuffle" { w.outputs = make([]*Output, len(w.inputs)) for i, input := range w.inputs { w.outputs[i] = new(Output) w.outputs[i].output_location = input.replica_location w.outputs[i].label = input.label w.outputs[i].output_size = 0 // TODO find a way to calculate the size } } else { w.runStage(pwd, "reduce_out_", Reduce) } send_output(w.outputs) request_done() }