func dirMapper(key interface{}, value interface{}) map[interface{}]interface{} {
	ret := make(map[interface{}]interface{})

	files, ok := value.([]interface{})
	if !ok {
		return ret
	}

	runFileChan := make(chan map[interface{}]interface{}, 100)
	go func() {
		defer close(runFileChan)

		var wg sync.WaitGroup
		job := core.NewJob(lineMapper, wordReducer)

		for _, v := range files {
			filename := strings.TrimSpace(v.(string))

			wg.Add(1)
			go func(myfile string) {
				defer wg.Done()

				runFileChan <- job.Run(file.ReadLines(myfile))
			}(filename)
		}
		wg.Wait()
	}()

	return core.Reduce(core.Aggregate(runFileChan), wordReducer)
}
示例#2
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	rand.Seed(int64(time.Second))

	ret := make(map[string]int)
	worker := core.NewWorker(MyProcesser, MyErrorHandler, 10, 10)
	out_chan, _ := worker.Run(file.ReadLines("wordcount.go"))
	for mapResult := range out_chan {
		m, ok := mapResult.(map[string]int)
		if !ok {
			continue
		}

		for k, v := range m {
			_, ok := ret[k]
			if ok {
				ret[k] += v
			} else {
				ret[k] = v
			}
		}
	}

	for k, v := range ret {
		println(fmt.Sprintf("%v          %v", k, v))
	}

}