func (w *Worker) runStage(pwd string, prefix string, process Process) {
output, err := ioutil.TempFile(pwd, prefix)
output_name := output.Name()
Check(err)
defer output.Close()
locations := make([]string, len(w.inputs))
for i, input := range w.inputs {
locations[i] = input.replica_location
}
readCloser := jobutil.AddressReader(locations, jobutil.Setting("DISCO_DATA"))
process(readCloser, output)
readCloser.Close()
fileinfo, err := output.Stat()
Check(err)
w.outputs = make([]*Output, 1)
w.outputs[0] = new(Output)
absDiscoPath, err := filepath.EvalSymlinks(w.task.Disco_data)
Check(err)
w.outputs[0].output_location =
"disco://" + jobutil.Setting("HOST") + "/disco/" + output_name[len(absDiscoPath)+1:]
w.outputs[0].output_size = fileinfo.Size()
}
func Post() {
master := "http://" + jobutil.Setting("DISCO_MASTER_HOST") + ":" + jobutil.Setting("DISCO_PORT")
response := submit_job(master)
defer response.Close()
body, err := ioutil.ReadAll(response)
Check(err)
result := make([]interface{}, 2)
err = json.Unmarshal(body, &result)
Check(err)
jobname := result[1].(string)
fmt.Println(jobname)
get_results(master, jobname)
}
func main() {
var master string
var confFile string
var inputs Inputs
var worker string
var jobtype string
const (
defaultMaster = "localhost"
masterUsage = "The master node."
defaultConf = "/etc/disco/settings.py"
confUsage = "The setting file which contains disco settings"
defaultWorker = ""
workerUsage = "The worker directory, a .go file, or an executable"
defaultInputs = ""
inputUsage = "The comma separated list of inputs to the job."
defaultJobType = "mapreduce"
jobTypeUsage = "type of the job (mapreduce or pipeline)"
)
flag.StringVar(&master, "Master", "", masterUsage)
flag.StringVar(&master, "M", "", masterUsage)
flag.StringVar(&confFile, "Conf", defaultConf, confUsage)
flag.StringVar(&confFile, "C", defaultConf, confUsage)
flag.StringVar(&worker, "Worker", defaultWorker, workerUsage)
flag.StringVar(&worker, "W", defaultWorker, workerUsage)
flag.Var(&inputs, "Inputs", inputUsage)
flag.Var(&inputs, "I", inputUsage)
flag.StringVar(&jobtype, "Type", defaultJobType, jobTypeUsage)
flag.StringVar(&jobtype, "T", defaultJobType, jobTypeUsage)
flag.Parse()
if worker == "" || len(inputs) == 0 {
fmt.Println("Usage: jobpack -W worker_dir -I input(s)")
os.Exit(1)
}
jobutil.AddFile(confFile)
if master != "" {
jobutil.SetKeyValue("DISCO_MASTER_HOST", master)
} else if jobutil.Setting("DISCO_MASTER_HOST") == "" {
jobutil.SetKeyValue("DISCO_MASTER_HOST", defaultMaster)
}
CreateJobPack(inputs, worker, jobtype)
Post()
Cleanup()
}
func get_results(master string, jobname string) {
outputs, err := jobutil.Wait(master, jobname, 20)
Check(err)
disco_root := jobutil.Setting("DISCO_ROOT")
readCloser := jobutil.AddressReader(outputs, disco_root+"/data")
defer readCloser.Close()
scanner := bufio.NewScanner(readCloser)
for scanner.Scan() {
fmt.Println(scanner.Text())
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
}
func get_results(master string, jobname string) {
outputs, err := jobutil.Wait(master, jobname, 20)
Check(err)
disco_root := jobutil.Setting("DISCO_ROOT")
readCloser := jobutil.AddressReader(outputs, disco_root+"/data")
defer readCloser.Close()
reader := bufio.NewReader(readCloser)
err = nil
line := []byte("")
for err == nil {
thisRead, isPrefix, thisErr := reader.ReadLine()
err = thisErr
line = append(line, thisRead...)
if !isPrefix {
fmt.Println(string(line))
line = []byte("")
}
}
if err != io.EOF {
log.Fatal(err)
}
}