// Example of stream processing with multi operators applied
// src - CsvSource to load data file, emits each record as []string
// out - CsvSink to write result file, expects each entry as []string
// Operations applied to stream:
// 0. stream.From() - reads stream from CsvSource
// 1. stream.Map() - maps []string to to scientist type
// 2. stream.Filter() - filters out scientist.BornYear > 1938
// 3. stream.Map() - maps scientist value to []string
// 4. stram.To() - writes sream values to a CsvSink
// 5. stream.Open() - opens and executes stream operator and wait for
// completion.
func main() {
in := src.New().WithFile("./data.txt")
out := snk.New().WithFile("./result.txt")
stream := stream.New().From(in)
stream.Map(func(cs []string) scientist {
yr, _ := strconv.Atoi(cs[3])
return scientist{
FirstName: cs[1],
LastName: cs[0],
Title: cs[2],
BornYear: yr,
}
})
stream.Filter(func(cs scientist) bool {
if cs.BornYear > 1930 {
return true
}
return false
})
stream.Map(func(cs scientist) []string {
return []string{cs.FirstName, cs.LastName, cs.Title}
})
stream.To(out)
<-stream.Open() // wait for completion
}
// Example Word Count
// Applies multiple operations to a stream to map the words
// and apply a reductive step to count the words.
// Stream operations:
// 1. stream.From() - sets sets source to emit each title as a string
// 2. stream.FlatMap() - Further splits each words into []slice
// 3. stream.Map() - maps each word to KV where KV[0] = word, KV[1]= occurence
// 4. stream.GrouBy() - reduces stream item and group them Key
// 5. stream.Restream() - re-emit grouped items as streamed items.
// 6. stream.Map() - maps incoming item to []string{key, totalCount}
// 7. stream.To() prints items to a file.
func main() {
src := stream.NewSliceSource("Hello World", "Hello Milkyway", "Hello Universe")
snk := csv.New().WithFile("./wc.out")
stream := stream.New().From(src)
stream.FlatMap(func(line string) []string {
return strings.Split(line, " ")
})
stream.Map(func(data string) tuple.KV {
return tuple.KV{data, 1}
})
stream.GroupBy(0).ReStream()
stream.Map(func(m tuple.KV) []string {
key := m[0].(string)
sum := sum(m[1].([]interface{}))
return []string{key, strconv.Itoa(sum)}
})
stream.To(snk)
<-stream.Open()
}