func urlLooper(sf types.SetupFunction, feeder chan<- interface{}, done types.AtomicBool) { in := sf.AsConsumer("MimeSplitterHtml").Build() for url := range in { if !done.Get() { u := ToString(url) feeder <- u } } }
func wordPrinter(sf types.SetupFunction, done chan struct{}) { defer close(done) in := sf.AsConsumer("github.com/apoydence/hydra/examples/wordCount.FinalWordCounter").Build() for wordMap := range in { for k, v := range ToMap(wordMap) { println(k, v) } } }
func textDownloader(sf types.SetupFunction, download string, mb int, done chan struct{}, closer func()) { in := sf.AsConsumer("MimeSplitterText").Build() var totalSize int64 = 0 for url := range in { if totalSize >= int64(mb)*1024*1024 { closer() continue } u := ToString(url) println("download", u, path.Base(u)) totalSize += saveToFile(Download(u), path.Join(download, path.Base(u))) println("Size", totalSize) } close(done) }