func (m *DefaultWorker) Map(args *util.MapInput, reply *int) error {
*reply = 1
m.logger.Println("Job is running : " + strconv.FormatInt(args.JobID, 10))
resultfile := util.Get_map_resultfile(args.Source, args.JobID)
_, err := os.Create(resultfile)
if err != nil {
m.logger.Println(err)
return err
}
var buf bytes.Buffer
infile, err := os.Open(util.Get_tmpfile(args.Source, args.JobID))
defer infile.Close()
if err != nil {
m.logger.Println(err)
return err
}
scanner := bufio.NewScanner(infile)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
buf.WriteString(scanner.Text() + ":1\n")
}
if err := scanner.Err(); err != nil {
m.logger.Println(err)
return err
}
ioutil.WriteFile(resultfile, buf.Bytes(), os.ModeAppend)
*reply = 0
return nil
}
func (m *Master) Split() {
file, err := os.Open(m.input)
defer file.Close()
if err != nil {
m.logger.Fatal(err)
}
fileinfo, _ := file.Stat()
filesize := fileinfo.Size()
splitsize := filesize / int64(m.workersize)
m.logger.Print(splitsize)
for i := int64(0); i < m.workersize; i++ {
size := splitsize
if i == m.workersize-1 {
size = filesize - i*splitsize
}
partBuffer := make([]byte, size)
file.Read(partBuffer)
tmpfile := util.Get_tmpfile(m.input, i)
_, err := os.Create(tmpfile)
if err != nil {
m.logger.Fatal(err)
}
ioutil.WriteFile(tmpfile, partBuffer, os.ModeAppend)
m.logger.Print("Split to file:" + tmpfile)
}
}