func (pipeline *Pipeline) Partition(fn stream.PredicateFn) (*Pipeline, *Pipeline) {
lhsIn, lhsOut := stream.New(pipeline.Stream.Capacity())
rhsIn := dispatchers.New(pipeline.Context).If(fn).Dispatch(pipeline.Stream, lhsOut)
lhsPipeline := &Pipeline{Context: pipeline.Context, Stream: lhsIn, parallel: pipeline.parallel}
rhsPipeline := &Pipeline{Context: pipeline.Context, Stream: rhsIn, parallel: pipeline.parallel}
return lhsPipeline, rhsPipeline
}
func TestBatchCacheLoader(t *testing.T) {
Convey("Given I have an empty batch of size 2", t, func() {
batch := &appx.MemcacheLoadBatch{Size: 2}
So(batch.Empty(), ShouldBeTrue)
So(batch.Full(), ShouldBeFalse)
Convey("When I add an entity to the batch", func() {
batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
Convey("Then the batch is no longer empty", func() {
So(batch.Empty(), ShouldBeFalse)
Convey("And it is not yet full", func() {
So(batch.Full(), ShouldBeFalse)
})
})
})
Convey("When I add enough entities", func() {
batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
batch.Add(NewUserWithFakeKey(User{Name: "diego"}))
Convey("Then the batch is full", func() {
So(batch.Full(), ShouldBeTrue)
})
})
Convey("When I commit the batch", func() {
in, out := stream.New(1)
entity1 := NewUserWithFakeKey(User{
Name: "entity1",
SSN: "123123",
})
entity2 := NewUserWithFakeKey(User{
Name: "entity2",
SSN: "321321",
})
batch.Add(entity1)
batch.Add(entity2)
batch.Commit(stream.NewEmitter(rivers.NewContext(), out))
close(out)
Convey("Then a copy of the batch is sent to the output stream", func() {
committedBatch := (<-in).(*appx.MemcacheLoadBatch)
So(committedBatch.Size, ShouldEqual, 2)
So(committedBatch.Keys[0], ShouldEqual, entity1.CacheID())
So(committedBatch.Keys[1], ShouldEqual, entity2.CacheID())
So(committedBatch.Items[entity1.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity1})
So(committedBatch.Items[entity2.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity2})
Convey("And the batch is now empty", func() {
So(batch.Empty(), ShouldBeTrue)
})
})
})
})
}
func TestZipperBy(t *testing.T) {
adder := func(a, b stream.T) stream.T {
return a.(int) + b.(int)
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in1, out1 := stream.New(2)
out1 <- 1
out1 <- 2
close(out1)
in2, out2 := stream.New(4)
out2 <- 3
out2 <- 4
out2 <- 5
out2 <- 6
close(out2)
Convey("When I apply the combiner to the streams", func() {
combiner := combiners.ZipBy(adder)
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then a transformed stream is returned", func() {
So(combined.ReadAll(), ShouldResemble, []stream.T{4, 6, 5, 6})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
combiner := combiners.ZipBy(adder)
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then no item is sent to the next stage", func() {
So(combined.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func (dispatcher *dispatcher) Dispatch(in stream.Readable, writables ...stream.Writable) stream.Readable {
notDispatchedReadable, notDispatchedWritable := stream.New(in.Capacity())
dispatchedCount := 0
done := make(chan bool, len(writables))
closeWritables := func() {
defer func() {
for _, writable := range writables {
close(writable)
}
}()
expectedDoneMessages := dispatchedCount * len(writables)
for i := 0; i < expectedDoneMessages; i++ {
select {
case <-dispatcher.context.Failure():
return
case <-time.After(dispatcher.context.Deadline()):
panic(stream.Timeout)
case <-done:
continue
}
}
}
go func() {
defer dispatcher.context.Recover()
defer close(notDispatchedWritable)
defer closeWritables()
for data := range in {
select {
case <-dispatcher.context.Failure():
return
case <-time.After(dispatcher.context.Deadline()):
panic(stream.Timeout)
default:
if dispatcher.fn(data) {
dispatchedCount++
for _, writable := range writables {
// dispatch data asynchronously so that
// slow receivers don't block the dispatch
// process
go func(w stream.Writable, d stream.T) {
w <- d
done <- true
}(writable, data)
}
} else {
notDispatchedWritable <- data
}
}
}
}()
return notDispatchedReadable
}
func TestFifo(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in1, out1 := stream.New(2)
out1 <- 1
out1 <- 2
close(out1)
in2, out2 := stream.New(2)
out2 <- 3
out2 <- 4
close(out2)
Convey("When I apply the combiner to the streams", func() {
combiner := combiners.FIFO()
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then a transformed stream is returned", func() {
items := combined.ReadAll()
So(items, should.Contain, 1)
So(items, should.Contain, 2)
So(items, should.Contain, 3)
So(items, should.Contain, 4)
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
combiner := combiners.Zip()
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then no item is sent to the next stage", func() {
So(combined.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func (combiner *zipBy) Combine(in ...stream.Readable) stream.Readable {
max := func(rs ...stream.Readable) int {
max := 0
for _, r := range rs {
capacity := r.Capacity()
if max < capacity {
max = capacity
}
}
return max
}
reader, writer := stream.New(max(in...))
go func() {
defer combiner.context.Recover()
defer close(writer)
var zipped stream.T
doneIndexes := make(map[int]bool)
for len(doneIndexes) < len(in) {
select {
case <-combiner.context.Failure():
return
case <-time.After(combiner.context.Deadline()):
panic(stream.Timeout)
default:
for i, readable := range in {
data, opened := <-readable
if !opened {
if _, registered := doneIndexes[i]; !registered {
doneIndexes[i] = true
}
continue
}
if zipped == nil {
zipped = data
} else {
zipped = combiner.fn(zipped, data)
}
}
if zipped != nil {
writer <- zipped
zipped = nil
}
}
}
}()
return reader
}
func TestEach(t *testing.T) {
collect := func(items *[]stream.T) stream.EachFn {
return func(data stream.T) {
*items = append(*items, data)
}
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the transformer to the stream", func() {
var items []stream.T
transformer := transformers.Each(collect(&items))
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then all items are sent to the next stage", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{1, 2})
Convey("And all items are transformed", func() {
So(items, ShouldResemble, []stream.T{1, 2})
})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
var items []stream.T
transformer := transformers.Each(collect(&items))
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
Convey("And no item is transformed", func() {
So(items, ShouldBeEmpty)
})
})
})
})
})
})
}
func (pipeline *Pipeline) SplitN(n int) []*Pipeline {
pipelines := make([]*Pipeline, n)
writables := make([]stream.Writable, n)
for i := 0; i < n; i++ {
readable, writable := stream.New(pipeline.Stream.Capacity())
writables[i] = writable
pipelines[i] = &Pipeline{
Context: pipeline.Context,
Stream: readable,
parallel: pipeline.parallel,
}
}
dispatchers.New(pipeline.Context).Always().Dispatch(pipeline.Stream, writables...)
return pipelines
}
func TestBatcher(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply the batch transformer to the stream", func() {
transformer := transformers.Batch(2)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1, 2}, []stream.T{3}})
})
})
Convey("When I apply the batch by transformer to the stream", func() {
transformer := transformers.BatchBy(&batch{size: 1})
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1}, []stream.T{2}, []stream.T{3}})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Flatten()
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func (observable *Observable) Produce() stream.Readable {
if observable.Capacity <= 0 {
observable.Capacity = 10
}
readable, writable := stream.New(observable.Capacity)
go func() {
defer observable.context.Recover()
defer close(writable)
if observable.Emit != nil {
observable.Emit(stream.NewEmitter(observable.context, writable))
}
}()
return readable
}
func (observer *Observer) Transform(in stream.Readable) stream.Readable {
readable, writable := stream.New(in.Capacity())
emitter := stream.NewEmitter(observer.context, writable)
go func() {
defer observer.context.Recover()
defer close(writable)
for {
select {
case <-observer.context.Failure():
return
case <-observer.context.Done():
return
case <-time.After(observer.context.Deadline()):
panic(stream.Timeout)
default:
data, more := <-in
if !more {
if observer.OnCompleted != nil {
observer.OnCompleted(emitter)
}
return
}
if observer.OnNext == nil {
continue
}
if err := observer.OnNext(data, emitter); err != nil {
if err == stream.Done {
// Tell producer to shutdown without errors
observer.context.Close(nil)
return
}
panic(err)
}
}
}
}()
return readable
}
func TestProcessor(t *testing.T) {
evensFilter := func(d stream.T, emitter stream.Emitter) {
if d.(int)%2 == 0 {
emitter.Emit(d)
}
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.OnData(evensFilter)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.OnData(evensFilter)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func (combiner *zip) Combine(in ...stream.Readable) stream.Readable {
capacity := func(rs ...stream.Readable) int {
capacity := 0
for _, r := range rs {
capacity += r.Capacity()
}
return capacity
}
reader, writer := stream.New(capacity(in...))
go func() {
defer combiner.context.Recover()
defer close(writer)
for {
select {
case <-combiner.context.Failure():
return
case <-time.After(combiner.context.Deadline()):
panic(stream.Timeout)
default:
doneCount := 0
for _, readable := range in {
data, more := <-readable
if !more {
doneCount++
continue
}
writer <- data
}
if doneCount == len(in) {
return
}
}
}
}()
return reader
}
func TestFindBy(t *testing.T) {
evens := func(d stream.T) bool { return d.(int)%2 == 0 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 4
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.FindBy(evens)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.FindBy(evens)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestReducer(t *testing.T) {
sum := func(acc, next stream.T) stream.T { return acc.(int) + next.(int) }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply a mapper transformer to the stream", func() {
transformer := transformers.Reduce(0, sum)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{6})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Reduce(0, sum)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestItemsCollector(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the collector consumer", func() {
var data []stream.T
consumer := consumers.ItemsCollector(&data)
consumer.Attach(context)
consumer.Consume(in)
Convey("Then data is collected out of the stream", func() {
So(data, ShouldResemble, []stream.T{1, 2})
data, opened := <-in
So(data, ShouldBeNil)
So(opened, ShouldBeFalse)
})
})
Convey("When I apply the collector consuming data into a non slice pointer", func() {
var data []stream.T
collect := func() {
consumers.ItemsCollector(data)
}
Convey("Then it panics", func() {
So(collect, ShouldPanicWith, consumers.ErrNoSuchSlicePointer)
})
})
})
})
}
func (combiner *fifo) Combine(in ...stream.Readable) stream.Readable {
capacity := func(in ...stream.Readable) int {
capacity := 0
for _, r := range in {
capacity += r.Capacity()
}
return capacity
}
var wg sync.WaitGroup
reader, writer := stream.New(capacity(in...))
for _, r := range in {
wg.Add(1)
go func(r stream.Readable) {
defer combiner.context.Recover()
defer wg.Done()
select {
case <-combiner.context.Failure():
return
case <-time.After(combiner.context.Deadline()):
panic(stream.Timeout)
default:
for data := range r {
writer <- data
}
}
}(r)
}
go func() {
defer close(writer)
wg.Wait()
}()
return reader
}
func TestMapper(t *testing.T) {
inc := func(d stream.T) stream.T { return d.(int) + 1 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply a mapper transformer to the stream", func() {
transformer := transformers.Map(inc)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{2, 3})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Map(inc)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestTakeN(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.TakeFirst(2)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{1, 2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.TakeFirst(1)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestLastItemCollector(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the collector consumer", func() {
var number int
consumer := consumers.LastItemCollector(&number)
consumer.Attach(context)
consumer.Consume(in)
Convey("Then data is collected out of the stream", func() {
So(number, ShouldResemble, 2)
_, opened := <-in
So(opened, ShouldBeFalse)
})
})
Convey("When I apply the collector consuming data into a non pointer", func() {
var number int
collect := func() {
consumers.LastItemCollector(number)
}
Convey("Then it panics", func() {
So(collect, ShouldPanicWith, consumers.ErrNoSuchPointer)
})
})
})
})
}
func TestDrainer(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the drainer consumer", func() {
consumer := consumers.Drainer()
consumer.Attach(context)
consumer.Consume(in)
Convey("Then the stream is drained", func() {
data, opened := <-in
So(data, ShouldBeNil)
So(opened, ShouldBeFalse)
})
})
})
})
}
func TestIfDispatcher(t *testing.T) {
evens := func(d stream.T) bool { return d.(int)%2 == 0 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 2
out <- 3
out <- 4
close(out)
Convey("When I apply an if dispatcher", func() {
evensIn, evensOut := stream.New(3)
sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)
Convey("Then items matching the condition are dispatched to the corresponding stream", func() {
data := evensIn.ReadAll()
So(data, should.Contain, 2)
So(data, should.Contain, 4)
Convey("And items not matching the condition are dispatched to the sink stream", func() {
So(sink.ReadAll(), ShouldResemble, []stream.T{3})
})
})
})
Convey("When I apply an always dispatcher", func() {
streamIn1, streamOut1 := stream.New(3)
streamIn2, streamOut2 := stream.New(3)
sink := dispatchers.New(context).Always().Dispatch(in, streamOut1, streamOut2)
Convey("Then all items are dispatched to the corresponding streams", func() {
streamIn1Items := streamIn1.ReadAll()
streamIn2Items := streamIn2.ReadAll()
So(streamIn1Items, should.Contain, 2)
So(streamIn1Items, should.Contain, 3)
So(streamIn1Items, should.Contain, 4)
So(streamIn2Items, should.Contain, 2)
So(streamIn2Items, should.Contain, 3)
So(streamIn2Items, should.Contain, 4)
Convey("And no item is dispatched to the sink stream", func() {
So(sink.ReadAll(), ShouldBeEmpty)
})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
evensIn, evensOut := stream.New(3)
sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)
Convey("Then no item is sent to the next stage", func() {
So(evensIn.ReadAll(), ShouldBeEmpty)
So(sink.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestRiversAPI(t *testing.T) {
evensOnly := func(data stream.T) bool { return data.(int)%2 == 0 }
sum := func(a, b stream.T) stream.T { return a.(int) + b.(int) }
add := func(n int) stream.MapFn {
return func(data stream.T) stream.T { return data.(int) + n }
}
concat := func(c string) stream.MapFn {
return func(data stream.T) stream.T { return data.(string) + c }
}
addOrAppend := func(n int, c string) stream.MapFn {
return func(data stream.T) stream.T {
if num, ok := data.(int); ok {
return num + n
}
if letter, ok := data.(string); ok {
return letter + "_"
}
return data
}
}
alphabeticOrder := func(a, b stream.T) bool {
return a.(string) < b.(string)
}
Convey("rivers API", t, func() {
Convey("From Range -> Filter -> Map -> Reduce -> Each", func() {
data, _ := rivers.FromRange(1, 5).
Filter(evensOnly).
Map(add(1)).
Reduce(0, sum).
Collect()
So(data, ShouldResemble, []stream.T{8})
})
Convey("From Data -> Flatten -> Map -> Sort By", func() {
items, err := rivers.FromData([]stream.T{"a", "c"}, "b", []stream.T{"d", "e"}).
Flatten().
Map(concat("_")).
SortBy(alphabeticOrder)
So(err, ShouldBeNil)
So(items, ShouldResemble, []stream.T{"a_", "b_", "c_", "d_", "e_"})
})
Convey("From Data -> FlatMap", func() {
data, _ := rivers.FromRange(1, 3).
FlatMap(func(data stream.T) stream.T { return []stream.T{data, data.(int) + 1} }).
Collect()
So(data, ShouldResemble, []stream.T{1, 2, 2, 3, 3, 4})
})
Convey("From Slice -> Dispatch If -> Map", func() {
in, out := stream.New(2)
notDispatched, _ := rivers.FromSlice([]stream.T{1, 2, 3, 4, 5}).
DispatchIf(evensOnly, out).
Map(add(2)).
Collect()
data := in.ReadAll()
So(data, ShouldContain, 2)
So(data, ShouldContain, 4)
So(notDispatched, ShouldContain, 3)
So(notDispatched, ShouldContain, 5)
So(notDispatched, ShouldContain, 7)
})
Convey("Zip -> Map", func() {
numbers := rivers.FromData(1, 2, 3, 4)
letters := rivers.FromData("a", "b", "c")
combined, _ := numbers.Zip(letters).Map(addOrAppend(1, "_")).Collect()
So(combined, ShouldResemble, []stream.T{2, "a_", 3, "b_", 4, "c_", 5})
})
Convey("Zip By -> Filter -> Collect", func() {
numbers := rivers.FromData(1, 2, 3, 4)
moreNumbers := rivers.FromData(4, 4, 1)
combined, err := numbers.ZipBy(sum, moreNumbers).Filter(evensOnly).Collect()
So(err, ShouldBeNil)
So(combined, ShouldResemble, []stream.T{6, 4, 4})
})
Convey("Merge -> Map", func() {
numbers := rivers.FromData(1, 2)
moreNumbers := rivers.FromData(3, 4)
combined, _ := numbers.Merge(moreNumbers).Collect()
So(len(combined), ShouldEqual, 4)
So(combined, ShouldContain, 1)
So(combined, ShouldContain, 2)
So(combined, ShouldContain, 3)
So(combined, ShouldContain, 4)
})
Convey("From Data -> Drain", func() {
numbers := rivers.FromData(1, 2, 3, 4)
numbers.Drain()
data, opened := <-numbers.Stream
So(data, ShouldBeNil)
So(opened, ShouldBeFalse)
})
Convey("From Range -> Partition", func() {
evensStage, oddsStage := rivers.FromRange(1, 4).Partition(evensOnly)
evens, _ := evensStage.Collect()
odds, _ := oddsStage.Collect()
So(evens, ShouldContain, 2)
So(evens, ShouldContain, 4)
So(odds, ShouldContain, 1)
So(odds, ShouldContain, 3)
})
Convey("From Range -> Slipt", func() {
lhs, rhs := rivers.FromRange(1, 2).Split()
lhsData := lhs.Stream.ReadAll()
rhsData := rhs.Stream.ReadAll()
So(lhsData, ShouldContain, 1)
So(lhsData, ShouldContain, 2)
So(rhsData, ShouldContain, 1)
So(rhsData, ShouldContain, 2)
})
Convey("From Range -> Slipt N", func() {
pipelines := rivers.FromRange(1, 2).SplitN(3)
data0 := pipelines[0].Stream.ReadAll()
data1 := pipelines[1].Stream.ReadAll()
data2 := pipelines[2].Stream.ReadAll()
So(data0, ShouldContain, 1)
So(data0, ShouldContain, 2)
So(data1, ShouldContain, 1)
So(data1, ShouldContain, 2)
So(data2, ShouldContain, 1)
So(data2, ShouldContain, 2)
})
Convey("From Range -> OnData", func() {
pipeline := rivers.FromRange(1, 4).OnData(func(data stream.T, emitter stream.Emitter) {
if data.(int)%2 == 0 {
emitter.Emit(data)
}
})
So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4})
})
Convey("From Range -> TakeFirst N -> Collect", func() {
taken, _ := rivers.FromRange(1, 4).TakeFirst(2).Collect()
So(taken, ShouldResemble, []stream.T{1, 2})
})
Convey("From Range -> Take", func() {
pipeline := rivers.FromRange(1, 4).Take(evensOnly)
So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4})
})
Convey("From Range -> Drop", func() {
pipeline := rivers.FromRange(1, 4).Drop(evensOnly)
So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{1, 3})
})
Convey("From Range -> Drop First 2", func() {
pipeline := rivers.FromRange(1, 5).DropFirst(2)
So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{3, 4, 5})
})
Convey("From Range -> Collect", func() {
data, err := rivers.FromRange(1, 4).Collect()
So(err, ShouldBeNil)
So(data, ShouldResemble, []stream.T{1, 2, 3, 4})
})
Convey("From Range -> CollectFirst", func() {
data, err := rivers.FromRange(1, 4).CollectFirst()
So(err, ShouldBeNil)
So(data, ShouldEqual, 1)
})
Convey("From Range -> CollectFirstAs", func() {
var data int
err := rivers.FromRange(1, 4).CollectFirstAs(&data)
So(err, ShouldBeNil)
So(data, ShouldEqual, 1)
})
Convey("From Range -> CollectLast", func() {
data, err := rivers.FromRange(1, 4).CollectLast()
So(err, ShouldBeNil)
So(data, ShouldEqual, 4)
})
Convey("From Range -> CollectLastAs", func() {
var data int
err := rivers.FromRange(1, 4).CollectLastAs(&data)
So(err, ShouldBeNil)
So(data, ShouldEqual, 4)
})
Convey("From Range -> CollectAs", func() {
var numbers []int
err := rivers.FromRange(1, 4).CollectAs(&numbers)
So(err, ShouldBeNil)
So(numbers, ShouldResemble, []int{1, 2, 3, 4})
})
Convey("From Data -> Map From Struct To JSON", func() {
type Account struct{ Name string }
items := rivers.FromData(Account{"Diego"}).Map(from.StructToJSON).Stream.ReadAll()
So(items, ShouldResemble, []stream.T{[]byte(`{"Name":"Diego"}`)})
})
Convey("From Data -> Map From JSON To Struct", func() {
type Account struct{ Name string }
items := rivers.FromData([]byte(`{"Name":"Diego"}`)).Map(from.JSONToStruct(Account{})).Stream.ReadAll()
So(items, ShouldResemble, []stream.T{Account{"Diego"}})
})
Convey("From Range -> Collect By", func() {
items := []stream.T{}
err := rivers.FromRange(1, 5).CollectBy(func(data stream.T) {
items = append(items, data)
})
So(err, ShouldBeNil)
So(items, ShouldResemble, []stream.T{1, 2, 3, 4, 5})
})
Convey("From Range -> Find", func() {
data, err := rivers.FromRange(1, 5).Find(2).Collect()
So(err, ShouldBeNil)
So(data, ShouldResemble, []stream.T{2})
})
Convey("From Range -> Find By", func() {
data, err := rivers.FromRange(1, 5).FindBy(func(subject stream.T) bool { return subject == 2 }).Collect()
So(err, ShouldBeNil)
So(data, ShouldResemble, []stream.T{2})
})
Convey("From Range -> Parallel -> Each", func() {
start := time.Now()
err := rivers.FromRange(1, 10).Parallel().Each(func(data stream.T) {
time.Sleep(500 * time.Millisecond)
}).Drain()
end := time.Since(start)
So(err, ShouldBeNil)
So(end.Seconds(), ShouldBeLessThanOrEqualTo, 1)
})
Convey("From Slow Producer -> Find", func() {
slowProducer := &producers.Observable{
Capacity: 2,
Emit: func(emitter stream.Emitter) {
for i := 0; i < 5; i++ {
emitter.Emit(i)
time.Sleep(300 * time.Millisecond)
}
},
}
items, err := rivers.From(slowProducer).Find(2).Collect()
So(err, ShouldBeNil)
So(items, ShouldResemble, []stream.T{2})
})
Convey("Producer times out", func() {
slowProducer := &producers.Observable{
Capacity: 2,
Emit: func(emitter stream.Emitter) {
for i := 0; i < 5; i++ {
emitter.Emit(i)
time.Sleep(2 * time.Second)
}
},
}
items, err := rivers.From(slowProducer).Deadline(300 * time.Millisecond).Find(2).Collect()
So(err, ShouldEqual, stream.Timeout)
So(items, ShouldBeNil)
})
Convey("Transformer times out", func() {
slowTransformer := &transformers.Observer{
OnNext: func(data stream.T, emitter stream.Emitter) error {
time.Sleep(2 * time.Second)
emitter.Emit(data)
return nil
},
}
items, err := rivers.FromRange(1, 3).Deadline(300 * time.Millisecond).Apply(slowTransformer).Collect()
So(err, ShouldEqual, stream.Timeout)
So(items, ShouldBeNil)
})
Convey("From Range -> Group By", func() {
evensAndOdds := func(data stream.T) (key stream.T) {
if data.(int)%2 == 0 {
return "evens"
}
return "odds"
}
groups, err := rivers.FromRange(1, 5).GroupBy(evensAndOdds)
So(err, ShouldBeNil)
So(groups.Empty(), ShouldBeFalse)
So(groups.HasGroup("evens"), ShouldBeTrue)
So(groups.HasGroup("odds"), ShouldBeTrue)
So(groups.HasGroup("invalid"), ShouldBeFalse)
So(groups.HasItem(2), ShouldBeTrue)
So(groups.HasItem(6), ShouldBeFalse)
So(groups, ShouldResemble, stream.Groups{
"evens": []stream.T{2, 4},
"odds": []stream.T{1, 3, 5},
})
})
Convey("From Reader -> Map -> Filter -> Collect", func() {
toString := func(data stream.T) stream.T { return string(data.(byte)) }
dashes := func(data stream.T) bool { return data == "-" }
items, err := rivers.FromReader(bytes.NewReader([]byte("abcd"))).Map(toString).Drop(dashes).Collect()
So(err, ShouldBeNil)
So(items, ShouldResemble, []stream.T{"a", "b", "c", "d"})
})
Convey("From Range -> Count", func() {
count, err := rivers.FromRange(1, 5).Count()
So(err, ShouldBeNil)
So(count, ShouldEqual, 5)
})
})
}
func TestLoadBatchFromCache(t *testing.T) {
gaeCtx, _ := aetest.NewContext(nil)
defer gaeCtx.Close()
Convey("Given I have a load batch from cache transformer", t, func() {
riversCtx := rivers.NewContext()
loadBatchProcessor := appx.NewStep(riversCtx).LoadBatchFromCache(gaeCtx)
Convey("And I have a few entities in the cache", func() {
user1 := NewUser(User{
Name: "Borges",
Email: "*****@*****.**",
SSN: "123123123",
})
user2 := NewUser(User{
Name: "Diego",
Email: "*****@*****.**",
SSN: "321321",
})
appx.NewKeyResolver(gaeCtx).Resolve(user1)
appx.NewKeyResolver(gaeCtx).Resolve(user2)
memcache.JSON.Set(gaeCtx, &memcache.Item{
Key: user1.CacheID(),
Object: appx.CachedEntity{
Entity: user1,
Key: user1.Key(),
},
})
memcache.JSON.Set(gaeCtx, &memcache.Item{
Key: user2.CacheID(),
Object: appx.CachedEntity{
Entity: user2,
Key: user2.Key(),
},
})
Convey("When I transform the incoming batch", func() {
notCachedUser := NewUser(User{
Name: "not cached",
SSN: "notcached",
})
userFromCache1 := NewUser(User{Name: user1.Name})
userFromCache2 := NewUser(User{Name: user2.Name})
batchItems := make(map[string]*appx.CachedEntity)
batchItems[user1.CacheID()] = &appx.CachedEntity{
Entity: userFromCache1,
}
batchItems[user2.CacheID()] = &appx.CachedEntity{
Entity: userFromCache2,
}
batchItems[notCachedUser.CacheID()] = &appx.CachedEntity{
Entity: notCachedUser,
}
batch := &appx.MemcacheLoadBatch{
Keys: []string{user1.CacheID(), user2.CacheID()},
Items: batchItems,
}
in, out := stream.New(1)
loadBatchProcessor(batch, stream.NewEmitter(rivers.NewContext(), out))
close(out)
Convey("Then cache misses are sent downstream", func() {
So(in.ReadAll(), ShouldResemble, []stream.T{notCachedUser})
Convey("And entities are loaded from cache", func() {
So(userFromCache1, ShouldResemble, user1)
So(userFromCache2, ShouldResemble, user2)
})
})
})
})
})
}