func (m *DefaultWorker) Map(args *util.MapInput, reply *int) error { *reply = 1 m.logger.Println("Job is running : " + strconv.FormatInt(args.JobID, 10)) resultfile := util.Get_map_resultfile(args.Source, args.JobID) _, err := os.Create(resultfile) if err != nil { m.logger.Println(err) return err } var buf bytes.Buffer infile, err := os.Open(util.Get_tmpfile(args.Source, args.JobID)) defer infile.Close() if err != nil { m.logger.Println(err) return err } scanner := bufio.NewScanner(infile) scanner.Split(bufio.ScanWords) for scanner.Scan() { buf.WriteString(scanner.Text() + ":1\n") } if err := scanner.Err(); err != nil { m.logger.Println(err) return err } ioutil.WriteFile(resultfile, buf.Bytes(), os.ModeAppend) *reply = 0 return nil }
func (m *Master) Split() { file, err := os.Open(m.input) defer file.Close() if err != nil { m.logger.Fatal(err) } fileinfo, _ := file.Stat() filesize := fileinfo.Size() splitsize := filesize / int64(m.workersize) m.logger.Print(splitsize) for i := int64(0); i < m.workersize; i++ { size := splitsize if i == m.workersize-1 { size = filesize - i*splitsize } partBuffer := make([]byte, size) file.Read(partBuffer) tmpfile := util.Get_tmpfile(m.input, i) _, err := os.Create(tmpfile) if err != nil { m.logger.Fatal(err) } ioutil.WriteFile(tmpfile, partBuffer, os.ModeAppend) m.logger.Print("Split to file:" + tmpfile) } }