func dirMapper(key interface{}, value interface{}) map[interface{}]interface{} { ret := make(map[interface{}]interface{}) files, ok := value.([]interface{}) if !ok { return ret } runFileChan := make(chan map[interface{}]interface{}, 100) go func() { defer close(runFileChan) var wg sync.WaitGroup job := core.NewJob(lineMapper, wordReducer) for _, v := range files { filename := strings.TrimSpace(v.(string)) wg.Add(1) go func(myfile string) { defer wg.Done() runFileChan <- job.Run(file.ReadLines(myfile)) }(filename) } wg.Wait() }() return core.Reduce(core.Aggregate(runFileChan), wordReducer) }
func main() { runtime.GOMAXPROCS(runtime.NumCPU()) rand.Seed(int64(time.Second)) job := core.NewJob(dirMapper, wordReducer) result := job.Run(file.ReadDir("./", "")) core.PrintMap(result) }