Exemple #1
0
func (m *Master) Merge() {
	fo, err := os.Create(m.output)
	if err != nil {
		m.logger.Fatal(err)
	}
	defer fo.Close()

	buf := make([]byte, 1024)
	for i := range m.reducekeys {
		fi, err := os.Open(util.Get_reduce_resultfile(m.input, int64(i)))
		if err != nil {
			m.logger.Fatal(err)
		}
		defer fi.Close()
		for {
			n, err := fi.Read(buf)
			if err != nil && err != io.EOF {
				m.logger.Fatal(err)
			}

			if n == 0 {
				break
			}

			_, err = fo.Write(buf[:n])
			if err != nil {
				m.logger.Fatal(err)
			}
		}

	}
}
Exemple #2
0
func (m *Master) RunJob() {
	var wg sync.WaitGroup
	jobs := make(chan int64)
	go func() {
		for i := int64(0); i < m.workersize; i++ {
			jobs <- i
		}
	}()
	count := 0
	for i := range jobs {
		w := <-m.workerchan
		m.mapresult = append(m.mapresult, util.Get_map_resultfile(m.input, int64(i)))
		wg.Add(1)
		go func(jobid int64, w *util.WorkerInfo) {
			defer wg.Done()
			client, err := rpc.DialHTTP("tcp", w.Addr)
			if err != nil {
				m.logger.Fatal(err)
			}
			var reply int
			input := &util.MapInput{m.input, jobid}
			err = client.Call("DefaultWorker.Map", input, &reply)
			if err != nil {
				m.logger.Println(err)
				jobs <- jobid
				return
			}
			count += 1
			m.workerchan <- w
			if int64(count) == m.workersize {
				close(jobs)
			}
		}(i, w)
	}
	wg.Wait()

	keys := make(chan string)

	go func() {
		for _, key := range m.reducekeys {
			keys <- key
		}
	}()

	i := 0
	count = 0
	for key := range keys {
		w := <-m.workerchan
		wg.Add(1)
		go func(key string, i int, w *util.WorkerInfo) {
			defer wg.Done()
			client, err := rpc.DialHTTP("tcp", w.Addr)
			if err != nil {
				m.logger.Fatal(err)
			}
			var reply int
			err = client.Call("DefaultWorker.Reduce", &util.ReduceInput{m.mapresult, key, util.Get_reduce_resultfile(m.input, int64(i))}, &reply)
			if err != nil {
				m.logger.Println(err)
				keys <- key
				return
			}
			m.workerchan <- w
			count += 1
			if count == len(m.reducekeys) {
				close(keys)
			}

		}(key, i, w)
		i += 1
	}
	wg.Wait()
	m.logger.Println("MR done")
}