Exemplo n.º 1
0
func (w *Worker) runStage(pwd string, prefix string, process Process) {
	output, err := ioutil.TempFile(pwd, prefix)
	output_name := output.Name()
	Check(err)
	defer output.Close()
	locations := make([]string, len(w.inputs))
	for i, input := range w.inputs {
		locations[i] = input.replica_location
	}

	readCloser := jobutil.AddressReader(locations, jobutil.Setting("DISCO_DATA"))
	process(readCloser, output)
	readCloser.Close()

	fileinfo, err := output.Stat()
	Check(err)

	w.outputs = make([]*Output, 1)
	w.outputs[0] = new(Output)

	absDiscoPath, err := filepath.EvalSymlinks(w.task.Disco_data)
	Check(err)
	w.outputs[0].output_location =
		"disco://" + jobutil.Setting("HOST") + "/disco/" + output_name[len(absDiscoPath)+1:]
	w.outputs[0].output_size = fileinfo.Size()
}
Exemplo n.º 2
0
func Post() {
	master := "http://" + jobutil.Setting("DISCO_MASTER_HOST") + ":" + jobutil.Setting("DISCO_PORT")
	response := submit_job(master)
	defer response.Close()
	body, err := ioutil.ReadAll(response)
	Check(err)

	result := make([]interface{}, 2)
	err = json.Unmarshal(body, &result)
	Check(err)
	jobname := result[1].(string)
	fmt.Println(jobname)
	get_results(master, jobname)
}
Exemplo n.º 3
0
func main() {
	var master string
	var confFile string
	var inputs Inputs
	var worker string
	var jobtype string

	const (
		defaultMaster  = "localhost"
		masterUsage    = "The master node."
		defaultConf    = "/etc/disco/settings.py"
		confUsage      = "The setting file which contains disco settings"
		defaultWorker  = ""
		workerUsage    = "The worker directory, a .go file, or an executable"
		defaultInputs  = ""
		inputUsage     = "The comma separated list of inputs to the job."
		defaultJobType = "mapreduce"
		jobTypeUsage   = "type of the job (mapreduce or pipeline)"
	)
	flag.StringVar(&master, "Master", "", masterUsage)
	flag.StringVar(&master, "M", "", masterUsage)
	flag.StringVar(&confFile, "Conf", defaultConf, confUsage)
	flag.StringVar(&confFile, "C", defaultConf, confUsage)
	flag.StringVar(&worker, "Worker", defaultWorker, workerUsage)
	flag.StringVar(&worker, "W", defaultWorker, workerUsage)

	flag.Var(&inputs, "Inputs", inputUsage)
	flag.Var(&inputs, "I", inputUsage)

	flag.StringVar(&jobtype, "Type", defaultJobType, jobTypeUsage)
	flag.StringVar(&jobtype, "T", defaultJobType, jobTypeUsage)

	flag.Parse()

	if worker == "" || len(inputs) == 0 {
		fmt.Println("Usage: jobpack -W worker_dir -I input(s)")
		os.Exit(1)
	}

	jobutil.AddFile(confFile)
	if master != "" {
		jobutil.SetKeyValue("DISCO_MASTER_HOST", master)
	} else if jobutil.Setting("DISCO_MASTER_HOST") == "" {
		jobutil.SetKeyValue("DISCO_MASTER_HOST", defaultMaster)
	}

	CreateJobPack(inputs, worker, jobtype)
	Post()
	Cleanup()
}
Exemplo n.º 4
0
func get_results(master string, jobname string) {
	outputs, err := jobutil.Wait(master, jobname, 20)
	Check(err)

	disco_root := jobutil.Setting("DISCO_ROOT")
	readCloser := jobutil.AddressReader(outputs, disco_root+"/data")
	defer readCloser.Close()
	scanner := bufio.NewScanner(readCloser)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}
Exemplo n.º 5
0
func get_results(master string, jobname string) {
	outputs, err := jobutil.Wait(master, jobname, 20)
	Check(err)

	disco_root := jobutil.Setting("DISCO_ROOT")
	readCloser := jobutil.AddressReader(outputs, disco_root+"/data")
	defer readCloser.Close()

	reader := bufio.NewReader(readCloser)
	err = nil
	line := []byte("")
	for err == nil {
		thisRead, isPrefix, thisErr := reader.ReadLine()
		err = thisErr
		line = append(line, thisRead...)
		if !isPrefix {
			fmt.Println(string(line))
			line = []byte("")
		}
	}
	if err != io.EOF {
		log.Fatal(err)
	}
}