// Example of stream processing with multi operators applied // src - CsvSource to load data file, emits each record as []string // out - CsvSink to write result file, expects each entry as []string // Operations applied to stream: // 0. stream.From() - reads stream from CsvSource // 1. stream.Map() - maps []string to to scientist type // 2. stream.Filter() - filters out scientist.BornYear > 1938 // 3. stream.Map() - maps scientist value to []string // 4. stram.To() - writes sream values to a CsvSink // 5. stream.Open() - opens and executes stream operator and wait for // completion. func main() { in := src.New().WithFile("./data.txt") out := snk.New().WithFile("./result.txt") stream := stream.New().From(in) stream.Map(func(cs []string) scientist { yr, _ := strconv.Atoi(cs[3]) return scientist{ FirstName: cs[1], LastName: cs[0], Title: cs[2], BornYear: yr, } }) stream.Filter(func(cs scientist) bool { if cs.BornYear > 1930 { return true } return false }) stream.Map(func(cs scientist) []string { return []string{cs.FirstName, cs.LastName, cs.Title} }) stream.To(out) <-stream.Open() // wait for completion }
// Example Word Count // Applies multiple operations to a stream to map the words // and apply a reductive step to count the words. // Stream operations: // 1. stream.From() - sets sets source to emit each title as a string // 2. stream.FlatMap() - Further splits each words into []slice // 3. stream.Map() - maps each word to KV where KV[0] = word, KV[1]= occurence // 4. stream.GrouBy() - reduces stream item and group them Key // 5. stream.Restream() - re-emit grouped items as streamed items. // 6. stream.Map() - maps incoming item to []string{key, totalCount} // 7. stream.To() prints items to a file. func main() { src := stream.NewSliceSource("Hello World", "Hello Milkyway", "Hello Universe") snk := csv.New().WithFile("./wc.out") stream := stream.New().From(src) stream.FlatMap(func(line string) []string { return strings.Split(line, " ") }) stream.Map(func(data string) tuple.KV { return tuple.KV{data, 1} }) stream.GroupBy(0).ReStream() stream.Map(func(m tuple.KV) []string { key := m[0].(string) sum := sum(m[1].([]interface{})) return []string{key, strconv.Itoa(sum)} }) stream.To(snk) <-stream.Open() }