Example #1
0
func (pipeline *Pipeline) Partition(fn stream.PredicateFn) (*Pipeline, *Pipeline) {
	lhsIn, lhsOut := stream.New(pipeline.Stream.Capacity())
	rhsIn := dispatchers.New(pipeline.Context).If(fn).Dispatch(pipeline.Stream, lhsOut)
	lhsPipeline := &Pipeline{Context: pipeline.Context, Stream: lhsIn, parallel: pipeline.parallel}
	rhsPipeline := &Pipeline{Context: pipeline.Context, Stream: rhsIn, parallel: pipeline.parallel}
	return lhsPipeline, rhsPipeline
}
Example #2
0
func TestBatchCacheLoader(t *testing.T) {
	Convey("Given I have an empty batch of size 2", t, func() {
		batch := &appx.MemcacheLoadBatch{Size: 2}
		So(batch.Empty(), ShouldBeTrue)
		So(batch.Full(), ShouldBeFalse)

		Convey("When I add an entity to the batch", func() {
			batch.Add(NewUserWithFakeKey(User{Name: "borges"}))

			Convey("Then the batch is no longer empty", func() {
				So(batch.Empty(), ShouldBeFalse)

				Convey("And it is not yet full", func() {
					So(batch.Full(), ShouldBeFalse)
				})
			})
		})

		Convey("When I add enough entities", func() {
			batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
			batch.Add(NewUserWithFakeKey(User{Name: "diego"}))

			Convey("Then the batch is full", func() {
				So(batch.Full(), ShouldBeTrue)
			})
		})

		Convey("When I commit the batch", func() {
			in, out := stream.New(1)

			entity1 := NewUserWithFakeKey(User{
				Name: "entity1",
				SSN:  "123123",
			})

			entity2 := NewUserWithFakeKey(User{
				Name: "entity2",
				SSN:  "321321",
			})

			batch.Add(entity1)
			batch.Add(entity2)
			batch.Commit(stream.NewEmitter(rivers.NewContext(), out))
			close(out)

			Convey("Then a copy of the batch is sent to the output stream", func() {
				committedBatch := (<-in).(*appx.MemcacheLoadBatch)
				So(committedBatch.Size, ShouldEqual, 2)
				So(committedBatch.Keys[0], ShouldEqual, entity1.CacheID())
				So(committedBatch.Keys[1], ShouldEqual, entity2.CacheID())
				So(committedBatch.Items[entity1.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity1})
				So(committedBatch.Items[entity2.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity2})

				Convey("And the batch is now empty", func() {
					So(batch.Empty(), ShouldBeTrue)
				})
			})
		})
	})
}
Example #3
0
func TestZipperBy(t *testing.T) {
	adder := func(a, b stream.T) stream.T {
		return a.(int) + b.(int)
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in1, out1 := stream.New(2)
			out1 <- 1
			out1 <- 2
			close(out1)

			in2, out2 := stream.New(4)
			out2 <- 3
			out2 <- 4
			out2 <- 5
			out2 <- 6
			close(out2)

			Convey("When I apply the combiner to the streams", func() {
				combiner := combiners.ZipBy(adder)
				combiner.Attach(context)
				combined := combiner.Combine(in1, in2)

				Convey("Then a transformed stream is returned", func() {
					So(combined.ReadAll(), ShouldResemble, []stream.T{4, 6, 5, 6})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					combiner := combiners.ZipBy(adder)
					combiner.Attach(context)
					combined := combiner.Combine(in1, in2)

					Convey("Then no item is sent to the next stage", func() {
						So(combined.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #4
0
func (dispatcher *dispatcher) Dispatch(in stream.Readable, writables ...stream.Writable) stream.Readable {
	notDispatchedReadable, notDispatchedWritable := stream.New(in.Capacity())

	dispatchedCount := 0
	done := make(chan bool, len(writables))

	closeWritables := func() {
		defer func() {
			for _, writable := range writables {
				close(writable)
			}
		}()

		expectedDoneMessages := dispatchedCount * len(writables)
		for i := 0; i < expectedDoneMessages; i++ {
			select {
			case <-dispatcher.context.Failure():
				return
			case <-time.After(dispatcher.context.Deadline()):
				panic(stream.Timeout)
			case <-done:
				continue
			}
		}
	}

	go func() {
		defer dispatcher.context.Recover()
		defer close(notDispatchedWritable)
		defer closeWritables()

		for data := range in {
			select {
			case <-dispatcher.context.Failure():
				return
			case <-time.After(dispatcher.context.Deadline()):
				panic(stream.Timeout)
			default:
				if dispatcher.fn(data) {
					dispatchedCount++
					for _, writable := range writables {
						// dispatch data asynchronously so that
						// slow receivers don't block the dispatch
						// process
						go func(w stream.Writable, d stream.T) {
							w <- d
							done <- true
						}(writable, data)
					}
				} else {
					notDispatchedWritable <- data
				}
			}
		}
	}()

	return notDispatchedReadable
}
Example #5
0
func TestFifo(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in1, out1 := stream.New(2)
			out1 <- 1
			out1 <- 2
			close(out1)

			in2, out2 := stream.New(2)
			out2 <- 3
			out2 <- 4
			close(out2)

			Convey("When I apply the combiner to the streams", func() {
				combiner := combiners.FIFO()
				combiner.Attach(context)
				combined := combiner.Combine(in1, in2)

				Convey("Then a transformed stream is returned", func() {
					items := combined.ReadAll()
					So(items, should.Contain, 1)
					So(items, should.Contain, 2)
					So(items, should.Contain, 3)
					So(items, should.Contain, 4)
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					combiner := combiners.Zip()
					combiner.Attach(context)
					combined := combiner.Combine(in1, in2)

					Convey("Then no item is sent to the next stage", func() {
						So(combined.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #6
0
func (combiner *zipBy) Combine(in ...stream.Readable) stream.Readable {
	max := func(rs ...stream.Readable) int {
		max := 0
		for _, r := range rs {
			capacity := r.Capacity()
			if max < capacity {
				max = capacity
			}
		}
		return max
	}

	reader, writer := stream.New(max(in...))

	go func() {
		defer combiner.context.Recover()
		defer close(writer)

		var zipped stream.T
		doneIndexes := make(map[int]bool)

		for len(doneIndexes) < len(in) {
			select {
			case <-combiner.context.Failure():
				return
			case <-time.After(combiner.context.Deadline()):
				panic(stream.Timeout)
			default:
				for i, readable := range in {
					data, opened := <-readable

					if !opened {
						if _, registered := doneIndexes[i]; !registered {
							doneIndexes[i] = true
						}
						continue
					}

					if zipped == nil {
						zipped = data
					} else {
						zipped = combiner.fn(zipped, data)
					}
				}

				if zipped != nil {
					writer <- zipped
					zipped = nil
				}
			}
		}
	}()

	return reader
}
Example #7
0
func TestEach(t *testing.T) {
	collect := func(items *[]stream.T) stream.EachFn {
		return func(data stream.T) {
			*items = append(*items, data)
		}
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				var items []stream.T
				transformer := transformers.Each(collect(&items))
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then all items are sent to the next stage", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{1, 2})

					Convey("And all items are transformed", func() {
						So(items, ShouldResemble, []stream.T{1, 2})
					})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					var items []stream.T
					transformer := transformers.Each(collect(&items))
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)

						Convey("And no item is transformed", func() {
							So(items, ShouldBeEmpty)
						})
					})
				})
			})
		})
	})
}
Example #8
0
func (pipeline *Pipeline) SplitN(n int) []*Pipeline {
	pipelines := make([]*Pipeline, n)
	writables := make([]stream.Writable, n)
	for i := 0; i < n; i++ {
		readable, writable := stream.New(pipeline.Stream.Capacity())
		writables[i] = writable
		pipelines[i] = &Pipeline{
			Context:  pipeline.Context,
			Stream:   readable,
			parallel: pipeline.parallel,
		}
	}
	dispatchers.New(pipeline.Context).Always().Dispatch(pipeline.Stream, writables...)
	return pipelines
}
Example #9
0
func TestBatcher(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 3
			close(out)

			Convey("When I apply the batch transformer to the stream", func() {
				transformer := transformers.Batch(2)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1, 2}, []stream.T{3}})
				})
			})

			Convey("When I apply the batch by transformer to the stream", func() {
				transformer := transformers.BatchBy(&batch{size: 1})
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1}, []stream.T{2}, []stream.T{3}})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.Flatten()
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #10
0
func (observable *Observable) Produce() stream.Readable {
	if observable.Capacity <= 0 {
		observable.Capacity = 10
	}
	readable, writable := stream.New(observable.Capacity)

	go func() {
		defer observable.context.Recover()
		defer close(writable)

		if observable.Emit != nil {
			observable.Emit(stream.NewEmitter(observable.context, writable))
		}
	}()

	return readable
}
Example #11
0
func (observer *Observer) Transform(in stream.Readable) stream.Readable {
	readable, writable := stream.New(in.Capacity())
	emitter := stream.NewEmitter(observer.context, writable)

	go func() {
		defer observer.context.Recover()
		defer close(writable)

		for {
			select {
			case <-observer.context.Failure():
				return
			case <-observer.context.Done():
				return
			case <-time.After(observer.context.Deadline()):
				panic(stream.Timeout)
			default:
				data, more := <-in
				if !more {
					if observer.OnCompleted != nil {
						observer.OnCompleted(emitter)
					}
					return
				}

				if observer.OnNext == nil {
					continue
				}

				if err := observer.OnNext(data, emitter); err != nil {
					if err == stream.Done {
						// Tell producer to shutdown without errors
						observer.context.Close(nil)
						return
					}
					panic(err)
				}
			}
		}
	}()

	return readable
}
Example #12
0
func TestProcessor(t *testing.T) {
	evensFilter := func(d stream.T, emitter stream.Emitter) {
		if d.(int)%2 == 0 {
			emitter.Emit(d)
		}
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				transformer := transformers.OnData(evensFilter)
				transformer.Attach(context)
				transformed := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(transformed.ReadAll(), ShouldResemble, []stream.T{2})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.OnData(evensFilter)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #13
0
func (combiner *zip) Combine(in ...stream.Readable) stream.Readable {
	capacity := func(rs ...stream.Readable) int {
		capacity := 0
		for _, r := range rs {
			capacity += r.Capacity()
		}
		return capacity
	}

	reader, writer := stream.New(capacity(in...))

	go func() {
		defer combiner.context.Recover()
		defer close(writer)

		for {
			select {
			case <-combiner.context.Failure():
				return
			case <-time.After(combiner.context.Deadline()):
				panic(stream.Timeout)
			default:
				doneCount := 0
				for _, readable := range in {
					data, more := <-readable
					if !more {
						doneCount++
						continue
					}
					writer <- data
				}

				if doneCount == len(in) {
					return
				}
			}
		}
	}()

	return reader
}
Example #14
0
func TestFindBy(t *testing.T) {
	evens := func(d stream.T) bool { return d.(int)%2 == 0 }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 4
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				transformer := transformers.FindBy(evens)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{2})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.FindBy(evens)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #15
0
func TestReducer(t *testing.T) {
	sum := func(acc, next stream.T) stream.T { return acc.(int) + next.(int) }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 3
			close(out)

			Convey("When I apply a mapper transformer to the stream", func() {
				transformer := transformers.Reduce(0, sum)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{6})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.Reduce(0, sum)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #16
0
func TestItemsCollector(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the collector consumer", func() {
				var data []stream.T
				consumer := consumers.ItemsCollector(&data)
				consumer.Attach(context)
				consumer.Consume(in)

				Convey("Then data is collected out of the stream", func() {
					So(data, ShouldResemble, []stream.T{1, 2})

					data, opened := <-in
					So(data, ShouldBeNil)
					So(opened, ShouldBeFalse)
				})
			})

			Convey("When I apply the collector consuming data into a non slice pointer", func() {
				var data []stream.T
				collect := func() {
					consumers.ItemsCollector(data)
				}

				Convey("Then it panics", func() {
					So(collect, ShouldPanicWith, consumers.ErrNoSuchSlicePointer)
				})
			})
		})
	})
}
Example #17
0
func (combiner *fifo) Combine(in ...stream.Readable) stream.Readable {
	capacity := func(in ...stream.Readable) int {
		capacity := 0
		for _, r := range in {
			capacity += r.Capacity()
		}
		return capacity
	}

	var wg sync.WaitGroup
	reader, writer := stream.New(capacity(in...))

	for _, r := range in {
		wg.Add(1)
		go func(r stream.Readable) {
			defer combiner.context.Recover()
			defer wg.Done()

			select {
			case <-combiner.context.Failure():
				return
			case <-time.After(combiner.context.Deadline()):
				panic(stream.Timeout)
			default:
				for data := range r {
					writer <- data
				}
			}
		}(r)
	}

	go func() {
		defer close(writer)
		wg.Wait()
	}()

	return reader
}
Example #18
0
func TestMapper(t *testing.T) {
	inc := func(d stream.T) stream.T { return d.(int) + 1 }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply a mapper transformer to the stream", func() {
				transformer := transformers.Map(inc)
				transformer.Attach(context)
				transformed := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(transformed.ReadAll(), ShouldResemble, []stream.T{2, 3})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.Map(inc)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #19
0
func TestTakeN(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 3
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				transformer := transformers.TakeFirst(2)
				transformer.Attach(context)
				transformed := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(transformed.ReadAll(), ShouldResemble, []stream.T{1, 2})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.TakeFirst(1)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
func TestLastItemCollector(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the collector consumer", func() {
				var number int
				consumer := consumers.LastItemCollector(&number)
				consumer.Attach(context)
				consumer.Consume(in)

				Convey("Then data is collected out of the stream", func() {
					So(number, ShouldResemble, 2)

					_, opened := <-in
					So(opened, ShouldBeFalse)
				})
			})

			Convey("When I apply the collector consuming data into a non pointer", func() {
				var number int
				collect := func() {
					consumers.LastItemCollector(number)
				}

				Convey("Then it panics", func() {
					So(collect, ShouldPanicWith, consumers.ErrNoSuchPointer)
				})
			})
		})
	})
}
Example #21
0
func TestDrainer(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the drainer consumer", func() {
				consumer := consumers.Drainer()
				consumer.Attach(context)
				consumer.Consume(in)

				Convey("Then the stream is drained", func() {
					data, opened := <-in
					So(data, ShouldBeNil)
					So(opened, ShouldBeFalse)
				})
			})
		})
	})
}
Example #22
0
func TestIfDispatcher(t *testing.T) {
	evens := func(d stream.T) bool { return d.(int)%2 == 0 }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 2
			out <- 3
			out <- 4
			close(out)

			Convey("When I apply an if dispatcher", func() {
				evensIn, evensOut := stream.New(3)
				sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)

				Convey("Then items matching the condition are dispatched to the corresponding stream", func() {
					data := evensIn.ReadAll()
					So(data, should.Contain, 2)
					So(data, should.Contain, 4)

					Convey("And items not matching the condition are dispatched to the sink stream", func() {
						So(sink.ReadAll(), ShouldResemble, []stream.T{3})
					})
				})
			})

			Convey("When I apply an always dispatcher", func() {
				streamIn1, streamOut1 := stream.New(3)
				streamIn2, streamOut2 := stream.New(3)
				sink := dispatchers.New(context).Always().Dispatch(in, streamOut1, streamOut2)

				Convey("Then all items are dispatched to the corresponding streams", func() {
					streamIn1Items := streamIn1.ReadAll()
					streamIn2Items := streamIn2.ReadAll()
					So(streamIn1Items, should.Contain, 2)
					So(streamIn1Items, should.Contain, 3)
					So(streamIn1Items, should.Contain, 4)

					So(streamIn2Items, should.Contain, 2)
					So(streamIn2Items, should.Contain, 3)
					So(streamIn2Items, should.Contain, 4)

					Convey("And no item is dispatched to the sink stream", func() {
						So(sink.ReadAll(), ShouldBeEmpty)
					})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					evensIn, evensOut := stream.New(3)
					sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)

					Convey("Then no item is sent to the next stage", func() {
						So(evensIn.ReadAll(), ShouldBeEmpty)
						So(sink.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}
Example #23
0
func TestRiversAPI(t *testing.T) {
	evensOnly := func(data stream.T) bool { return data.(int)%2 == 0 }
	sum := func(a, b stream.T) stream.T { return a.(int) + b.(int) }
	add := func(n int) stream.MapFn {
		return func(data stream.T) stream.T { return data.(int) + n }
	}

	concat := func(c string) stream.MapFn {
		return func(data stream.T) stream.T { return data.(string) + c }
	}

	addOrAppend := func(n int, c string) stream.MapFn {
		return func(data stream.T) stream.T {
			if num, ok := data.(int); ok {
				return num + n
			}
			if letter, ok := data.(string); ok {
				return letter + "_"
			}
			return data
		}
	}

	alphabeticOrder := func(a, b stream.T) bool {
		return a.(string) < b.(string)
	}

	Convey("rivers API", t, func() {

		Convey("From Range -> Filter -> Map -> Reduce -> Each", func() {
			data, _ := rivers.FromRange(1, 5).
				Filter(evensOnly).
				Map(add(1)).
				Reduce(0, sum).
				Collect()

			So(data, ShouldResemble, []stream.T{8})
		})

		Convey("From Data -> Flatten -> Map -> Sort By", func() {
			items, err := rivers.FromData([]stream.T{"a", "c"}, "b", []stream.T{"d", "e"}).
				Flatten().
				Map(concat("_")).
				SortBy(alphabeticOrder)

			So(err, ShouldBeNil)
			So(items, ShouldResemble, []stream.T{"a_", "b_", "c_", "d_", "e_"})
		})

		Convey("From Data -> FlatMap", func() {
			data, _ := rivers.FromRange(1, 3).
				FlatMap(func(data stream.T) stream.T { return []stream.T{data, data.(int) + 1} }).
				Collect()

			So(data, ShouldResemble, []stream.T{1, 2, 2, 3, 3, 4})
		})

		Convey("From Slice -> Dispatch If -> Map", func() {
			in, out := stream.New(2)

			notDispatched, _ := rivers.FromSlice([]stream.T{1, 2, 3, 4, 5}).
				DispatchIf(evensOnly, out).
				Map(add(2)).
				Collect()

			data := in.ReadAll()
			So(data, ShouldContain, 2)
			So(data, ShouldContain, 4)

			So(notDispatched, ShouldContain, 3)
			So(notDispatched, ShouldContain, 5)
			So(notDispatched, ShouldContain, 7)
		})

		Convey("Zip -> Map", func() {
			numbers := rivers.FromData(1, 2, 3, 4)
			letters := rivers.FromData("a", "b", "c")

			combined, _ := numbers.Zip(letters).Map(addOrAppend(1, "_")).Collect()

			So(combined, ShouldResemble, []stream.T{2, "a_", 3, "b_", 4, "c_", 5})
		})

		Convey("Zip By -> Filter -> Collect", func() {
			numbers := rivers.FromData(1, 2, 3, 4)
			moreNumbers := rivers.FromData(4, 4, 1)

			combined, err := numbers.ZipBy(sum, moreNumbers).Filter(evensOnly).Collect()

			So(err, ShouldBeNil)
			So(combined, ShouldResemble, []stream.T{6, 4, 4})
		})

		Convey("Merge -> Map", func() {
			numbers := rivers.FromData(1, 2)
			moreNumbers := rivers.FromData(3, 4)

			combined, _ := numbers.Merge(moreNumbers).Collect()

			So(len(combined), ShouldEqual, 4)
			So(combined, ShouldContain, 1)
			So(combined, ShouldContain, 2)
			So(combined, ShouldContain, 3)
			So(combined, ShouldContain, 4)
		})

		Convey("From Data -> Drain", func() {
			numbers := rivers.FromData(1, 2, 3, 4)
			numbers.Drain()

			data, opened := <-numbers.Stream
			So(data, ShouldBeNil)
			So(opened, ShouldBeFalse)
		})

		Convey("From Range -> Partition", func() {
			evensStage, oddsStage := rivers.FromRange(1, 4).Partition(evensOnly)
			evens, _ := evensStage.Collect()
			odds, _ := oddsStage.Collect()

			So(evens, ShouldContain, 2)
			So(evens, ShouldContain, 4)

			So(odds, ShouldContain, 1)
			So(odds, ShouldContain, 3)
		})

		Convey("From Range -> Slipt", func() {
			lhs, rhs := rivers.FromRange(1, 2).Split()

			lhsData := lhs.Stream.ReadAll()
			rhsData := rhs.Stream.ReadAll()
			So(lhsData, ShouldContain, 1)
			So(lhsData, ShouldContain, 2)

			So(rhsData, ShouldContain, 1)
			So(rhsData, ShouldContain, 2)
		})

		Convey("From Range -> Slipt N", func() {
			pipelines := rivers.FromRange(1, 2).SplitN(3)

			data0 := pipelines[0].Stream.ReadAll()
			data1 := pipelines[1].Stream.ReadAll()
			data2 := pipelines[2].Stream.ReadAll()

			So(data0, ShouldContain, 1)
			So(data0, ShouldContain, 2)

			So(data1, ShouldContain, 1)
			So(data1, ShouldContain, 2)

			So(data2, ShouldContain, 1)
			So(data2, ShouldContain, 2)
		})

		Convey("From Range -> OnData", func() {
			pipeline := rivers.FromRange(1, 4).OnData(func(data stream.T, emitter stream.Emitter) {
				if data.(int)%2 == 0 {
					emitter.Emit(data)
				}
			})

			So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4})
		})

		Convey("From Range -> TakeFirst N -> Collect", func() {
			taken, _ := rivers.FromRange(1, 4).TakeFirst(2).Collect()

			So(taken, ShouldResemble, []stream.T{1, 2})
		})

		Convey("From Range -> Take", func() {
			pipeline := rivers.FromRange(1, 4).Take(evensOnly)

			So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4})
		})

		Convey("From Range -> Drop", func() {
			pipeline := rivers.FromRange(1, 4).Drop(evensOnly)

			So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{1, 3})
		})

		Convey("From Range -> Drop First 2", func() {
			pipeline := rivers.FromRange(1, 5).DropFirst(2)

			So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{3, 4, 5})
		})

		Convey("From Range -> Collect", func() {
			data, err := rivers.FromRange(1, 4).Collect()

			So(err, ShouldBeNil)
			So(data, ShouldResemble, []stream.T{1, 2, 3, 4})
		})

		Convey("From Range -> CollectFirst", func() {
			data, err := rivers.FromRange(1, 4).CollectFirst()

			So(err, ShouldBeNil)
			So(data, ShouldEqual, 1)
		})

		Convey("From Range -> CollectFirstAs", func() {
			var data int
			err := rivers.FromRange(1, 4).CollectFirstAs(&data)

			So(err, ShouldBeNil)
			So(data, ShouldEqual, 1)
		})

		Convey("From Range -> CollectLast", func() {
			data, err := rivers.FromRange(1, 4).CollectLast()

			So(err, ShouldBeNil)
			So(data, ShouldEqual, 4)
		})

		Convey("From Range -> CollectLastAs", func() {
			var data int
			err := rivers.FromRange(1, 4).CollectLastAs(&data)

			So(err, ShouldBeNil)
			So(data, ShouldEqual, 4)
		})

		Convey("From Range -> CollectAs", func() {
			var numbers []int
			err := rivers.FromRange(1, 4).CollectAs(&numbers)

			So(err, ShouldBeNil)
			So(numbers, ShouldResemble, []int{1, 2, 3, 4})
		})

		Convey("From Data -> Map From Struct To JSON", func() {
			type Account struct{ Name string }

			items := rivers.FromData(Account{"Diego"}).Map(from.StructToJSON).Stream.ReadAll()

			So(items, ShouldResemble, []stream.T{[]byte(`{"Name":"Diego"}`)})
		})

		Convey("From Data -> Map From JSON To Struct", func() {
			type Account struct{ Name string }

			items := rivers.FromData([]byte(`{"Name":"Diego"}`)).Map(from.JSONToStruct(Account{})).Stream.ReadAll()

			So(items, ShouldResemble, []stream.T{Account{"Diego"}})
		})

		Convey("From Range -> Collect By", func() {
			items := []stream.T{}
			err := rivers.FromRange(1, 5).CollectBy(func(data stream.T) {
				items = append(items, data)
			})

			So(err, ShouldBeNil)
			So(items, ShouldResemble, []stream.T{1, 2, 3, 4, 5})
		})

		Convey("From Range -> Find", func() {
			data, err := rivers.FromRange(1, 5).Find(2).Collect()

			So(err, ShouldBeNil)
			So(data, ShouldResemble, []stream.T{2})
		})

		Convey("From Range -> Find By", func() {
			data, err := rivers.FromRange(1, 5).FindBy(func(subject stream.T) bool { return subject == 2 }).Collect()

			So(err, ShouldBeNil)
			So(data, ShouldResemble, []stream.T{2})
		})

		Convey("From Range -> Parallel -> Each", func() {
			start := time.Now()
			err := rivers.FromRange(1, 10).Parallel().Each(func(data stream.T) {
				time.Sleep(500 * time.Millisecond)
			}).Drain()
			end := time.Since(start)

			So(err, ShouldBeNil)
			So(end.Seconds(), ShouldBeLessThanOrEqualTo, 1)
		})

		Convey("From Slow Producer -> Find", func() {
			slowProducer := &producers.Observable{
				Capacity: 2,
				Emit: func(emitter stream.Emitter) {
					for i := 0; i < 5; i++ {
						emitter.Emit(i)
						time.Sleep(300 * time.Millisecond)
					}
				},
			}

			items, err := rivers.From(slowProducer).Find(2).Collect()
			So(err, ShouldBeNil)
			So(items, ShouldResemble, []stream.T{2})
		})

		Convey("Producer times out", func() {
			slowProducer := &producers.Observable{
				Capacity: 2,
				Emit: func(emitter stream.Emitter) {
					for i := 0; i < 5; i++ {
						emitter.Emit(i)
						time.Sleep(2 * time.Second)
					}
				},
			}

			items, err := rivers.From(slowProducer).Deadline(300 * time.Millisecond).Find(2).Collect()
			So(err, ShouldEqual, stream.Timeout)
			So(items, ShouldBeNil)
		})

		Convey("Transformer times out", func() {
			slowTransformer := &transformers.Observer{
				OnNext: func(data stream.T, emitter stream.Emitter) error {
					time.Sleep(2 * time.Second)
					emitter.Emit(data)
					return nil
				},
			}

			items, err := rivers.FromRange(1, 3).Deadline(300 * time.Millisecond).Apply(slowTransformer).Collect()
			So(err, ShouldEqual, stream.Timeout)
			So(items, ShouldBeNil)
		})

		Convey("From Range -> Group By", func() {
			evensAndOdds := func(data stream.T) (key stream.T) {
				if data.(int)%2 == 0 {
					return "evens"
				}

				return "odds"
			}

			groups, err := rivers.FromRange(1, 5).GroupBy(evensAndOdds)

			So(err, ShouldBeNil)
			So(groups.Empty(), ShouldBeFalse)
			So(groups.HasGroup("evens"), ShouldBeTrue)
			So(groups.HasGroup("odds"), ShouldBeTrue)
			So(groups.HasGroup("invalid"), ShouldBeFalse)
			So(groups.HasItem(2), ShouldBeTrue)
			So(groups.HasItem(6), ShouldBeFalse)
			So(groups, ShouldResemble, stream.Groups{
				"evens": []stream.T{2, 4},
				"odds":  []stream.T{1, 3, 5},
			})
		})

		Convey("From Reader -> Map -> Filter -> Collect", func() {
			toString := func(data stream.T) stream.T { return string(data.(byte)) }
			dashes := func(data stream.T) bool { return data == "-" }

			items, err := rivers.FromReader(bytes.NewReader([]byte("abcd"))).Map(toString).Drop(dashes).Collect()

			So(err, ShouldBeNil)
			So(items, ShouldResemble, []stream.T{"a", "b", "c", "d"})
		})

		Convey("From Range -> Count", func() {
			count, err := rivers.FromRange(1, 5).Count()

			So(err, ShouldBeNil)
			So(count, ShouldEqual, 5)
		})
	})
}
func TestLoadBatchFromCache(t *testing.T) {
	gaeCtx, _ := aetest.NewContext(nil)
	defer gaeCtx.Close()

	Convey("Given I have a load batch from cache transformer", t, func() {
		riversCtx := rivers.NewContext()
		loadBatchProcessor := appx.NewStep(riversCtx).LoadBatchFromCache(gaeCtx)

		Convey("And I have a few entities in the cache", func() {
			user1 := NewUser(User{
				Name:  "Borges",
				Email: "*****@*****.**",
				SSN:   "123123123",
			})

			user2 := NewUser(User{
				Name:  "Diego",
				Email: "*****@*****.**",
				SSN:   "321321",
			})

			appx.NewKeyResolver(gaeCtx).Resolve(user1)
			appx.NewKeyResolver(gaeCtx).Resolve(user2)

			memcache.JSON.Set(gaeCtx, &memcache.Item{
				Key: user1.CacheID(),
				Object: appx.CachedEntity{
					Entity: user1,
					Key:    user1.Key(),
				},
			})

			memcache.JSON.Set(gaeCtx, &memcache.Item{
				Key: user2.CacheID(),
				Object: appx.CachedEntity{
					Entity: user2,
					Key:    user2.Key(),
				},
			})

			Convey("When I transform the incoming batch", func() {
				notCachedUser := NewUser(User{
					Name: "not cached",
					SSN:  "notcached",
				})

				userFromCache1 := NewUser(User{Name: user1.Name})
				userFromCache2 := NewUser(User{Name: user2.Name})

				batchItems := make(map[string]*appx.CachedEntity)
				batchItems[user1.CacheID()] = &appx.CachedEntity{
					Entity: userFromCache1,
				}
				batchItems[user2.CacheID()] = &appx.CachedEntity{
					Entity: userFromCache2,
				}
				batchItems[notCachedUser.CacheID()] = &appx.CachedEntity{
					Entity: notCachedUser,
				}

				batch := &appx.MemcacheLoadBatch{
					Keys:  []string{user1.CacheID(), user2.CacheID()},
					Items: batchItems,
				}

				in, out := stream.New(1)
				loadBatchProcessor(batch, stream.NewEmitter(rivers.NewContext(), out))
				close(out)

				Convey("Then cache misses are sent downstream", func() {
					So(in.ReadAll(), ShouldResemble, []stream.T{notCachedUser})

					Convey("And entities are loaded from cache", func() {
						So(userFromCache1, ShouldResemble, user1)
						So(userFromCache2, ShouldResemble, user2)
					})
				})
			})
		})
	})
}