コード例 #1
0
ファイル: worker.go プロジェクト: maylingge/go-mapreduce
func (m *DefaultWorker) Map(args *util.MapInput, reply *int) error {
	*reply = 1
	m.logger.Println("Job is running : " + strconv.FormatInt(args.JobID, 10))
	resultfile := util.Get_map_resultfile(args.Source, args.JobID)
	_, err := os.Create(resultfile)
	if err != nil {
		m.logger.Println(err)
		return err
	}
	var buf bytes.Buffer

	infile, err := os.Open(util.Get_tmpfile(args.Source, args.JobID))
	defer infile.Close()
	if err != nil {
		m.logger.Println(err)
		return err
	}
	scanner := bufio.NewScanner(infile)
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		buf.WriteString(scanner.Text() + ":1\n")
	}
	if err := scanner.Err(); err != nil {
		m.logger.Println(err)
		return err
	}
	ioutil.WriteFile(resultfile, buf.Bytes(), os.ModeAppend)
	*reply = 0
	return nil
}
コード例 #2
0
ファイル: master.go プロジェクト: maylingge/go-mapreduce
func (m *Master) RunJob() {
	var wg sync.WaitGroup
	jobs := make(chan int64)
	go func() {
		for i := int64(0); i < m.workersize; i++ {
			jobs <- i
		}
	}()
	count := 0
	for i := range jobs {
		w := <-m.workerchan
		m.mapresult = append(m.mapresult, util.Get_map_resultfile(m.input, int64(i)))
		wg.Add(1)
		go func(jobid int64, w *util.WorkerInfo) {
			defer wg.Done()
			client, err := rpc.DialHTTP("tcp", w.Addr)
			if err != nil {
				m.logger.Fatal(err)
			}
			var reply int
			input := &util.MapInput{m.input, jobid}
			err = client.Call("DefaultWorker.Map", input, &reply)
			if err != nil {
				m.logger.Println(err)
				jobs <- jobid
				return
			}
			count += 1
			m.workerchan <- w
			if int64(count) == m.workersize {
				close(jobs)
			}
		}(i, w)
	}
	wg.Wait()

	keys := make(chan string)

	go func() {
		for _, key := range m.reducekeys {
			keys <- key
		}
	}()

	i := 0
	count = 0
	for key := range keys {
		w := <-m.workerchan
		wg.Add(1)
		go func(key string, i int, w *util.WorkerInfo) {
			defer wg.Done()
			client, err := rpc.DialHTTP("tcp", w.Addr)
			if err != nil {
				m.logger.Fatal(err)
			}
			var reply int
			err = client.Call("DefaultWorker.Reduce", &util.ReduceInput{m.mapresult, key, util.Get_reduce_resultfile(m.input, int64(i))}, &reply)
			if err != nil {
				m.logger.Println(err)
				keys <- key
				return
			}
			m.workerchan <- w
			count += 1
			if count == len(m.reducekeys) {
				close(keys)
			}

		}(key, i, w)
		i += 1
	}
	wg.Wait()
	m.logger.Println("MR done")
}