// An example Map function. It consumes json data and yields a value for each line
func (s *JsonEntryCounter) Mapper(r io.Reader, w io.Writer) error {
log.Printf("map_input_file %s", os.Getenv("map_input_file"))
wg, out := gomrjob.JsonInternalOutputProtocol(w)
for data := range gomrjob.JsonInputProtocol(r) {
gomrjob.Counter("example_mr", "Map Lines Read", 1)
key, err := data.Get(s.KeyField).String()
if err != nil {
gomrjob.Counter("example_mr", "Missing Key", 1)
} else {
out <- gomrjob.KeyValue{key, 1}
}
}
close(out)
wg.Wait()
return nil
}
// An example Map function. It consumes json data and yields a value for each line
func (s *JsonEntryCounter) Mapper(r io.Reader, w io.Writer) error {
log.Printf("map_input_file %s", os.Getenv("map_input_file"))
wg, out := gomrjob.JsonInternalOutputProtocol(w)
// for efficient counting, use an in-memory counter that flushes the least recently used item
// less Mapper output makes for faster sorting and reducing.
counter := lru.NewLRUCounter(func(k interface{}, v int64) {
out <- gomrjob.KeyValue{k, v}
}, 1)
for data := range gomrjob.JsonInputProtocol(r) {
gomrjob.Counter("example_mr", "Map Lines Read", 1)
key, err := data.Get(s.KeyField).String()
if err != nil {
gomrjob.Counter("example_mr", "Missing Key", 1)
} else {
counter.Incr(key, 1)
}
}
counter.Flush()
close(out)
wg.Wait()
return nil
}
