func dirMapper(key interface{}, value interface{}) map[interface{}]interface{} {
ret := make(map[interface{}]interface{})
files, ok := value.([]interface{})
if !ok {
return ret
}
runFileChan := make(chan map[interface{}]interface{}, 100)
go func() {
defer close(runFileChan)
var wg sync.WaitGroup
job := core.NewJob(lineMapper, wordReducer)
for _, v := range files {
filename := strings.TrimSpace(v.(string))
wg.Add(1)
go func(myfile string) {
defer wg.Done()
runFileChan <- job.Run(file.ReadLines(myfile))
}(filename)
}
wg.Wait()
}()
return core.Reduce(core.Aggregate(runFileChan), wordReducer)
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
rand.Seed(int64(time.Second))
ret := make(map[string]int)
worker := core.NewWorker(MyProcesser, MyErrorHandler, 10, 10)
out_chan, _ := worker.Run(file.ReadLines("wordcount.go"))
for mapResult := range out_chan {
m, ok := mapResult.(map[string]int)
if !ok {
continue
}
for k, v := range m {
_, ok := ret[k]
if ok {
ret[k] += v
} else {
ret[k] = v
}
}
}
for k, v := range ret {
println(fmt.Sprintf("%v %v", k, v))
}
}