func WordCounter(sf types.SetupFunction) { in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.SymbolRemover").Build() defer close(out) m := make(map[string]uint32) for word := range in { incMap(ToString(word), 1, m) } out <- NewWordCountMarshaler(m) }
func FinalWordCounter(sf types.SetupFunction) { in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordCounter").Build() defer close(out) m := make(map[string]uint32) for wordMap := range in { for k, v := range ToMap(wordMap) { incMap(k, v, m) } } out <- NewWordCountMarshaler(m) }
func SymbolRemover(sf types.SetupFunction) { in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordExtractor").Build() defer close(out) for word := range in { str := make([]byte, 0) bytes := []byte(strings.ToLower(ToString(word))) for _, x := range bytes { if (x >= 0x30 && x <= 0x39) || (x >= 0x61 && x <= 0x7a) { str = append(str, x) } } if len(str) > 0 { out <- NewStringMarshaler(string(str)) } } }
func WordExtractor(sf types.SetupFunction) { in, out := sf.AsFilter("PathValidator").Build() defer close(out) for path := range in { p := ToString(path) f, err := os.Open(p) if err == nil { scanner := bufio.NewScanner(f) scanner.Split(bufio.ScanWords) for scanner.Scan() { out <- NewStringMarshaler(scanner.Text()) } } } }