Beispiel #1
0
// Example of stream processing with multi operators applied
// src - CsvSource to load data file, emits each record as []string
// out - CsvSink to write result file, expects each entry as []string
// Operations applied to stream:
// 0. stream.From() - reads stream from CsvSource
// 1. stream.Map() - maps []string to to scientist type
// 2. stream.Filter() - filters out scientist.BornYear > 1938
// 3. stream.Map() - maps scientist value to []string
// 4. stram.To() - writes sream values to a CsvSink
// 5. stream.Open() - opens and executes stream operator and wait for
// completion.
func main() {
	in := src.New().WithFile("./data.txt")
	out := snk.New().WithFile("./result.txt")

	stream := stream.New().From(in)
	stream.Map(func(cs []string) scientist {
		yr, _ := strconv.Atoi(cs[3])
		return scientist{
			FirstName: cs[1],
			LastName:  cs[0],
			Title:     cs[2],
			BornYear:  yr,
		}
	})
	stream.Filter(func(cs scientist) bool {
		if cs.BornYear > 1930 {
			return true
		}
		return false
	})
	stream.Map(func(cs scientist) []string {
		return []string{cs.FirstName, cs.LastName, cs.Title}
	})
	stream.To(out)

	<-stream.Open() // wait for completion
}
Beispiel #2
0
// Example Word Count
// Applies multiple operations to a stream to map the words
// and apply a reductive step to count the words.
// Stream operations:
// 1. stream.From() - sets sets source to emit each title as a string
// 2. stream.FlatMap() - Further splits each words into []slice
// 3. stream.Map() - maps each word to KV where KV[0] = word, KV[1]= occurence
// 4. stream.GrouBy() - reduces stream item and group them Key
// 5. stream.Restream() - re-emit grouped items as streamed items.
// 6. stream.Map() - maps incoming item to []string{key, totalCount}
// 7. stream.To() prints items to a file.
func main() {
	src := stream.NewSliceSource("Hello World", "Hello Milkyway", "Hello Universe")
	snk := csv.New().WithFile("./wc.out")

	stream := stream.New().From(src)

	stream.FlatMap(func(line string) []string {
		return strings.Split(line, " ")
	})
	stream.Map(func(data string) tuple.KV {
		return tuple.KV{data, 1}
	})

	stream.GroupBy(0).ReStream()

	stream.Map(func(m tuple.KV) []string {
		key := m[0].(string)
		sum := sum(m[1].([]interface{}))
		return []string{key, strconv.Itoa(sum)}
	})

	stream.To(snk)
	<-stream.Open()
}