func (m *Master) Merge() {
fo, err := os.Create(m.output)
if err != nil {
m.logger.Fatal(err)
}
defer fo.Close()
buf := make([]byte, 1024)
for i := range m.reducekeys {
fi, err := os.Open(util.Get_reduce_resultfile(m.input, int64(i)))
if err != nil {
m.logger.Fatal(err)
}
defer fi.Close()
for {
n, err := fi.Read(buf)
if err != nil && err != io.EOF {
m.logger.Fatal(err)
}
if n == 0 {
break
}
_, err = fo.Write(buf[:n])
if err != nil {
m.logger.Fatal(err)
}
}
}
}
func (m *Master) RunJob() {
var wg sync.WaitGroup
jobs := make(chan int64)
go func() {
for i := int64(0); i < m.workersize; i++ {
jobs <- i
}
}()
count := 0
for i := range jobs {
w := <-m.workerchan
m.mapresult = append(m.mapresult, util.Get_map_resultfile(m.input, int64(i)))
wg.Add(1)
go func(jobid int64, w *util.WorkerInfo) {
defer wg.Done()
client, err := rpc.DialHTTP("tcp", w.Addr)
if err != nil {
m.logger.Fatal(err)
}
var reply int
input := &util.MapInput{m.input, jobid}
err = client.Call("DefaultWorker.Map", input, &reply)
if err != nil {
m.logger.Println(err)
jobs <- jobid
return
}
count += 1
m.workerchan <- w
if int64(count) == m.workersize {
close(jobs)
}
}(i, w)
}
wg.Wait()
keys := make(chan string)
go func() {
for _, key := range m.reducekeys {
keys <- key
}
}()
i := 0
count = 0
for key := range keys {
w := <-m.workerchan
wg.Add(1)
go func(key string, i int, w *util.WorkerInfo) {
defer wg.Done()
client, err := rpc.DialHTTP("tcp", w.Addr)
if err != nil {
m.logger.Fatal(err)
}
var reply int
err = client.Call("DefaultWorker.Reduce", &util.ReduceInput{m.mapresult, key, util.Get_reduce_resultfile(m.input, int64(i))}, &reply)
if err != nil {
m.logger.Println(err)
keys <- key
return
}
m.workerchan <- w
count += 1
if count == len(m.reducekeys) {
close(keys)
}
}(key, i, w)
i += 1
}
wg.Wait()
m.logger.Println("MR done")
}