func main() {
var master string
var confFile string
var inputs Inputs
var worker string
var jobtype string
const (
defaultMaster = "localhost"
masterUsage = "The master node."
defaultConf = "/etc/disco/settings.py"
confUsage = "The setting file which contains disco settings"
defaultWorker = ""
workerUsage = "The worker directory, a .go file, or an executable"
defaultInputs = ""
inputUsage = "The comma separated list of inputs to the job."
defaultJobType = "mapreduce"
jobTypeUsage = "type of the job (mapreduce or pipeline)"
)
flag.StringVar(&master, "Master", "", masterUsage)
flag.StringVar(&master, "M", "", masterUsage)
flag.StringVar(&confFile, "Conf", defaultConf, confUsage)
flag.StringVar(&confFile, "C", defaultConf, confUsage)
flag.StringVar(&worker, "Worker", defaultWorker, workerUsage)
flag.StringVar(&worker, "W", defaultWorker, workerUsage)
flag.Var(&inputs, "Inputs", inputUsage)
flag.Var(&inputs, "I", inputUsage)
flag.StringVar(&jobtype, "Type", defaultJobType, jobTypeUsage)
flag.StringVar(&jobtype, "T", defaultJobType, jobTypeUsage)
flag.Parse()
if worker == "" || len(inputs) == 0 {
fmt.Println("Usage: jobpack -W worker_dir -I input(s)")
os.Exit(1)
}
jobutil.AddFile(confFile)
if master != "" {
jobutil.SetKeyValue("DISCO_MASTER_HOST", master)
} else if jobutil.Setting("DISCO_MASTER_HOST") == "" {
jobutil.SetKeyValue("DISCO_MASTER_HOST", defaultMaster)
}
CreateJobPack(inputs, worker, jobtype)
Post()
Cleanup()
}
func Run(Map Process, Reduce Process) {
var w Worker
send_worker()
w.task = request_task()
jobutil.SetKeyValue("HOST", w.task.Host)
master, port := jobutil.HostAndPort(w.task.Master)
jobutil.SetKeyValue("DISCO_MASTER_HOST", master)
if port != fmt.Sprintf("%d", w.task.Disco_port) {
panic("port mismatch: " + port)
}
jobutil.SetKeyValue("DISCO_PORT", port)
jobutil.SetKeyValue("PUT_PORT", string(w.task.Put_port))
jobutil.SetKeyValue("DISCO_DATA", w.task.Disco_data)
jobutil.SetKeyValue("DDFS_DATA", w.task.Ddfs_data)
w.inputs = request_input()
pwd, err := os.Getwd()
Check(err)
if w.task.Stage == "map" {
w.runStage(pwd, "map_out_", Map)
} else if w.task.Stage == "map_shuffle" {
w.outputs = make([]*Output, len(w.inputs))
for i, input := range w.inputs {
w.outputs[i] = new(Output)
w.outputs[i].output_location = input.replica_location
w.outputs[i].label = input.label
w.outputs[i].output_size = 0 // TODO find a way to calculate the size
}
} else {
w.runStage(pwd, "reduce_out_", Reduce)
}
send_output(w.outputs)
request_done()
}
