// Source lists files under a hdfs folder, and process all files
// This is provided more as an example. You can copy the code and customize
// any way you want.
func Source(f *flow.FlowContext, hdfsLocation string, shard int) *flow.Dataset {
locations, err := List(hdfsLocation)
if err != nil {
log.Fatalf("Can not list files under %s:%v", hdfsLocation, err)
}
return f.Slice(locations).Partition(shard).Map(TextFile)
}
