func (w *Worker) runStage(pwd string, prefix string, process Process) { output, err := ioutil.TempFile(pwd, prefix) output_name := output.Name() Check(err) defer output.Close() locations := make([]string, len(w.inputs)) for i, input := range w.inputs { locations[i] = input.replica_location } readCloser := jobutil.AddressReader(locations, jobutil.Setting("DISCO_DATA")) process(readCloser, output) readCloser.Close() fileinfo, err := output.Stat() Check(err) w.outputs = make([]*Output, 1) w.outputs[0] = new(Output) absDiscoPath, err := filepath.EvalSymlinks(w.task.Disco_data) Check(err) w.outputs[0].output_location = "disco://" + jobutil.Setting("HOST") + "/disco/" + output_name[len(absDiscoPath)+1:] w.outputs[0].output_size = fileinfo.Size() }
func Post() { master := "http://" + jobutil.Setting("DISCO_MASTER_HOST") + ":" + jobutil.Setting("DISCO_PORT") response := submit_job(master) defer response.Close() body, err := ioutil.ReadAll(response) Check(err) result := make([]interface{}, 2) err = json.Unmarshal(body, &result) Check(err) jobname := result[1].(string) fmt.Println(jobname) get_results(master, jobname) }
func main() { var master string var confFile string var inputs Inputs var worker string var jobtype string const ( defaultMaster = "localhost" masterUsage = "The master node." defaultConf = "/etc/disco/settings.py" confUsage = "The setting file which contains disco settings" defaultWorker = "" workerUsage = "The worker directory, a .go file, or an executable" defaultInputs = "" inputUsage = "The comma separated list of inputs to the job." defaultJobType = "mapreduce" jobTypeUsage = "type of the job (mapreduce or pipeline)" ) flag.StringVar(&master, "Master", "", masterUsage) flag.StringVar(&master, "M", "", masterUsage) flag.StringVar(&confFile, "Conf", defaultConf, confUsage) flag.StringVar(&confFile, "C", defaultConf, confUsage) flag.StringVar(&worker, "Worker", defaultWorker, workerUsage) flag.StringVar(&worker, "W", defaultWorker, workerUsage) flag.Var(&inputs, "Inputs", inputUsage) flag.Var(&inputs, "I", inputUsage) flag.StringVar(&jobtype, "Type", defaultJobType, jobTypeUsage) flag.StringVar(&jobtype, "T", defaultJobType, jobTypeUsage) flag.Parse() if worker == "" || len(inputs) == 0 { fmt.Println("Usage: jobpack -W worker_dir -I input(s)") os.Exit(1) } jobutil.AddFile(confFile) if master != "" { jobutil.SetKeyValue("DISCO_MASTER_HOST", master) } else if jobutil.Setting("DISCO_MASTER_HOST") == "" { jobutil.SetKeyValue("DISCO_MASTER_HOST", defaultMaster) } CreateJobPack(inputs, worker, jobtype) Post() Cleanup() }
func get_results(master string, jobname string) { outputs, err := jobutil.Wait(master, jobname, 20) Check(err) disco_root := jobutil.Setting("DISCO_ROOT") readCloser := jobutil.AddressReader(outputs, disco_root+"/data") defer readCloser.Close() scanner := bufio.NewScanner(readCloser) for scanner.Scan() { fmt.Println(scanner.Text()) } if err := scanner.Err(); err != nil { log.Fatal(err) } }
func get_results(master string, jobname string) { outputs, err := jobutil.Wait(master, jobname, 20) Check(err) disco_root := jobutil.Setting("DISCO_ROOT") readCloser := jobutil.AddressReader(outputs, disco_root+"/data") defer readCloser.Close() reader := bufio.NewReader(readCloser) err = nil line := []byte("") for err == nil { thisRead, isPrefix, thisErr := reader.ReadLine() err = thisErr line = append(line, thisRead...) if !isPrefix { fmt.Println(string(line)) line = []byte("") } } if err != io.EOF { log.Fatal(err) } }