func WordCounter(sf types.SetupFunction) {
in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.SymbolRemover").Build()
defer close(out)
m := make(map[string]uint32)
for word := range in {
incMap(ToString(word), 1, m)
}
out <- NewWordCountMarshaler(m)
}
func FinalWordCounter(sf types.SetupFunction) {
in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordCounter").Build()
defer close(out)
m := make(map[string]uint32)
for wordMap := range in {
for k, v := range ToMap(wordMap) {
incMap(k, v, m)
}
}
out <- NewWordCountMarshaler(m)
}
func SymbolRemover(sf types.SetupFunction) {
in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordExtractor").Build()
defer close(out)
for word := range in {
str := make([]byte, 0)
bytes := []byte(strings.ToLower(ToString(word)))
for _, x := range bytes {
if (x >= 0x30 && x <= 0x39) || (x >= 0x61 && x <= 0x7a) {
str = append(str, x)
}
}
if len(str) > 0 {
out <- NewStringMarshaler(string(str))
}
}
}
func WordExtractor(sf types.SetupFunction) {
in, out := sf.AsFilter("PathValidator").Build()
defer close(out)
for path := range in {
p := ToString(path)
f, err := os.Open(p)
if err == nil {
scanner := bufio.NewScanner(f)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
out <- NewStringMarshaler(scanner.Text())
}
}
}
}
