func (m *DefaultWorker) Map(args *util.MapInput, reply *int) error {
*reply = 1
m.logger.Println("Job is running : " + strconv.FormatInt(args.JobID, 10))
resultfile := util.Get_map_resultfile(args.Source, args.JobID)
_, err := os.Create(resultfile)
if err != nil {
m.logger.Println(err)
return err
}
var buf bytes.Buffer
infile, err := os.Open(util.Get_tmpfile(args.Source, args.JobID))
defer infile.Close()
if err != nil {
m.logger.Println(err)
return err
}
scanner := bufio.NewScanner(infile)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
buf.WriteString(scanner.Text() + ":1\n")
}
if err := scanner.Err(); err != nil {
m.logger.Println(err)
return err
}
ioutil.WriteFile(resultfile, buf.Bytes(), os.ModeAppend)
*reply = 0
return nil
}
func (m *Master) RunJob() {
var wg sync.WaitGroup
jobs := make(chan int64)
go func() {
for i := int64(0); i < m.workersize; i++ {
jobs <- i
}
}()
count := 0
for i := range jobs {
w := <-m.workerchan
m.mapresult = append(m.mapresult, util.Get_map_resultfile(m.input, int64(i)))
wg.Add(1)
go func(jobid int64, w *util.WorkerInfo) {
defer wg.Done()
client, err := rpc.DialHTTP("tcp", w.Addr)
if err != nil {
m.logger.Fatal(err)
}
var reply int
input := &util.MapInput{m.input, jobid}
err = client.Call("DefaultWorker.Map", input, &reply)
if err != nil {
m.logger.Println(err)
jobs <- jobid
return
}
count += 1
m.workerchan <- w
if int64(count) == m.workersize {
close(jobs)
}
}(i, w)
}
wg.Wait()
keys := make(chan string)
go func() {
for _, key := range m.reducekeys {
keys <- key
}
}()
i := 0
count = 0
for key := range keys {
w := <-m.workerchan
wg.Add(1)
go func(key string, i int, w *util.WorkerInfo) {
defer wg.Done()
client, err := rpc.DialHTTP("tcp", w.Addr)
if err != nil {
m.logger.Fatal(err)
}
var reply int
err = client.Call("DefaultWorker.Reduce", &util.ReduceInput{m.mapresult, key, util.Get_reduce_resultfile(m.input, int64(i))}, &reply)
if err != nil {
m.logger.Println(err)
keys <- key
return
}
m.workerchan <- w
count += 1
if count == len(m.reducekeys) {
close(keys)
}
}(key, i, w)
i += 1
}
wg.Wait()
m.logger.Println("MR done")
}