func TestBatchCacheLoader(t *testing.T) {
Convey("Given I have an empty batch of size 2", t, func() {
batch := &appx.MemcacheLoadBatch{Size: 2}
So(batch.Empty(), ShouldBeTrue)
So(batch.Full(), ShouldBeFalse)
Convey("When I add an entity to the batch", func() {
batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
Convey("Then the batch is no longer empty", func() {
So(batch.Empty(), ShouldBeFalse)
Convey("And it is not yet full", func() {
So(batch.Full(), ShouldBeFalse)
})
})
})
Convey("When I add enough entities", func() {
batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
batch.Add(NewUserWithFakeKey(User{Name: "diego"}))
Convey("Then the batch is full", func() {
So(batch.Full(), ShouldBeTrue)
})
})
Convey("When I commit the batch", func() {
in, out := stream.New(1)
entity1 := NewUserWithFakeKey(User{
Name: "entity1",
SSN: "123123",
})
entity2 := NewUserWithFakeKey(User{
Name: "entity2",
SSN: "321321",
})
batch.Add(entity1)
batch.Add(entity2)
batch.Commit(stream.NewEmitter(rivers.NewContext(), out))
close(out)
Convey("Then a copy of the batch is sent to the output stream", func() {
committedBatch := (<-in).(*appx.MemcacheLoadBatch)
So(committedBatch.Size, ShouldEqual, 2)
So(committedBatch.Keys[0], ShouldEqual, entity1.CacheID())
So(committedBatch.Keys[1], ShouldEqual, entity2.CacheID())
So(committedBatch.Items[entity1.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity1})
So(committedBatch.Items[entity2.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity2})
Convey("And the batch is now empty", func() {
So(batch.Empty(), ShouldBeTrue)
})
})
})
})
}
func TestFromSlice(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And I have a slice producer", func() {
numbers := []int{1, 2, 3}
producer := producers.FromSlice(numbers)
Convey("When I produce data", func() {
producer.Attach(context)
readable := producer.Produce()
Convey("Then I can read the produced data from the stream", func() {
So(readable.ReadAll(), ShouldResemble, []stream.T{1, 2, 3})
})
})
})
Convey("And I have a data producer", func() {
producer := producers.FromData(1, 2, 3)
Convey("When I produce data", func() {
producer.Attach(context)
readable := producer.Produce()
Convey("Then I can read the produced data from the stream", func() {
So(readable.ReadAll(), ShouldResemble, []stream.T{1, 2, 3})
})
})
})
})
}
func TestLoadBatchFromDatastore(t *testing.T) {
gaeCtx, _ := aetest.NewContext(nil)
defer gaeCtx.Close()
Convey("Given I have a load batch from datastore transformer", t, func() {
riversCtx := rivers.NewContext()
loadBatchProcessor := appx.NewStep(riversCtx).LoadBatchFromDatastore(gaeCtx)
Convey("And I have a few entities in datastore", func() {
user1 := NewUser(User{
Name: "Borges",
Email: "*****@*****.**",
SSN: "123123123",
})
user2 := NewUser(User{
Name: "Borges",
Email: "*****@*****.**",
SSN: "123123123",
})
err := appx.NewKeyResolver(gaeCtx).Resolve(user1)
So(err, ShouldBeNil)
err = appx.NewKeyResolver(gaeCtx).Resolve(user2)
So(err, ShouldBeNil)
_, err = datastore.Put(gaeCtx, user1.Key(), user1)
So(err, ShouldBeNil)
_, err = datastore.Put(gaeCtx, user2.Key(), user2)
So(err, ShouldBeNil)
Convey("When I transform the incoming batch", func() {
userFromDatastore1 := NewUser(User{Name: user1.Name})
userFromDatastore2 := NewUser(User{Name: user2.Name})
appx.NewKeyResolver(gaeCtx).Resolve(userFromDatastore1)
appx.NewKeyResolver(gaeCtx).Resolve(userFromDatastore2)
batch := &appx.DatastoreBatch{
Size: 2,
Keys: []*datastore.Key{
userFromDatastore1.Key(),
userFromDatastore2.Key(),
},
Items: []appx.Entity{
userFromDatastore1,
userFromDatastore2,
},
}
loadBatchProcessor(batch)
Convey("And entities are loaded from datastore", func() {
So(userFromDatastore1, ShouldResemble, user1)
So(userFromDatastore2, ShouldResemble, user2)
})
})
})
})
}
func TestEach(t *testing.T) {
collect := func(items *[]stream.T) stream.EachFn {
return func(data stream.T) {
*items = append(*items, data)
}
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the transformer to the stream", func() {
var items []stream.T
transformer := transformers.Each(collect(&items))
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then all items are sent to the next stage", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{1, 2})
Convey("And all items are transformed", func() {
So(items, ShouldResemble, []stream.T{1, 2})
})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
var items []stream.T
transformer := transformers.Each(collect(&items))
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
Convey("And no item is transformed", func() {
So(items, ShouldBeEmpty)
})
})
})
})
})
})
}
func TestZipperBy(t *testing.T) {
adder := func(a, b stream.T) stream.T {
return a.(int) + b.(int)
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in1, out1 := stream.New(2)
out1 <- 1
out1 <- 2
close(out1)
in2, out2 := stream.New(4)
out2 <- 3
out2 <- 4
out2 <- 5
out2 <- 6
close(out2)
Convey("When I apply the combiner to the streams", func() {
combiner := combiners.ZipBy(adder)
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then a transformed stream is returned", func() {
So(combined.ReadAll(), ShouldResemble, []stream.T{4, 6, 5, 6})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
combiner := combiners.ZipBy(adder)
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then no item is sent to the next stage", func() {
So(combined.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestBatcher(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply the batch transformer to the stream", func() {
transformer := transformers.Batch(2)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1, 2}, []stream.T{3}})
})
})
Convey("When I apply the batch by transformer to the stream", func() {
transformer := transformers.BatchBy(&batch{size: 1})
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1}, []stream.T{2}, []stream.T{3}})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Flatten()
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestFifo(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in1, out1 := stream.New(2)
out1 <- 1
out1 <- 2
close(out1)
in2, out2 := stream.New(2)
out2 <- 3
out2 <- 4
close(out2)
Convey("When I apply the combiner to the streams", func() {
combiner := combiners.FIFO()
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then a transformed stream is returned", func() {
items := combined.ReadAll()
So(items, should.Contain, 1)
So(items, should.Contain, 2)
So(items, should.Contain, 3)
So(items, should.Contain, 4)
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
combiner := combiners.Zip()
combiner.Attach(context)
combined := combiner.Combine(in1, in2)
Convey("Then no item is sent to the next stage", func() {
So(combined.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestFromRange(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And I have a range producer", func() {
producer := producers.FromRange(1, 3)
producer.Attach(context)
Convey("When I produce data", func() {
readable := producer.Produce()
Convey("Then I can read the produced data from the stream", func() {
So(readable.ReadAll(), ShouldResemble, []stream.T{1, 2, 3})
})
})
})
})
}
func TestProcessor(t *testing.T) {
evensFilter := func(d stream.T, emitter stream.Emitter) {
if d.(int)%2 == 0 {
emitter.Emit(d)
}
}
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.OnData(evensFilter)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.OnData(evensFilter)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestFromFileByLine(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And I have a file with some data", func() {
ioutil.WriteFile("/tmp/from_file_by_line", []byte("Hello\nthere\nfolks!"), 0644)
file, _ := os.Open("/tmp/from_file_by_line")
Convey("When I produce data from the file", func() {
producer := producers.FromFile(file).ByLine()
producer.Attach(context)
readable := producer.Produce()
Convey("Then I can read the produced data from the stream", func() {
So(readable.ReadAll(), ShouldResemble, []stream.T{"Hello", "there", "folks!"})
})
})
})
})
}
func TestReducer(t *testing.T) {
sum := func(acc, next stream.T) stream.T { return acc.(int) + next.(int) }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply a mapper transformer to the stream", func() {
transformer := transformers.Reduce(0, sum)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{6})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Reduce(0, sum)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestFindBy(t *testing.T) {
evens := func(d stream.T) bool { return d.(int)%2 == 0 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 4
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.FindBy(evens)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(next.ReadAll(), ShouldResemble, []stream.T{2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.FindBy(evens)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestItemsCollector(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the collector consumer", func() {
var data []stream.T
consumer := consumers.ItemsCollector(&data)
consumer.Attach(context)
consumer.Consume(in)
Convey("Then data is collected out of the stream", func() {
So(data, ShouldResemble, []stream.T{1, 2})
data, opened := <-in
So(data, ShouldBeNil)
So(opened, ShouldBeFalse)
})
})
Convey("When I apply the collector consuming data into a non slice pointer", func() {
var data []stream.T
collect := func() {
consumers.ItemsCollector(data)
}
Convey("Then it panics", func() {
So(collect, ShouldPanicWith, consumers.ErrNoSuchSlicePointer)
})
})
})
})
}
func TestMapper(t *testing.T) {
inc := func(d stream.T) stream.T { return d.(int) + 1 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply a mapper transformer to the stream", func() {
transformer := transformers.Map(inc)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{2, 3})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.Map(inc)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestTakeN(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 1
out <- 2
out <- 3
close(out)
Convey("When I apply the transformer to the stream", func() {
transformer := transformers.TakeFirst(2)
transformer.Attach(context)
transformed := transformer.Transform(in)
Convey("Then a transformed stream is returned", func() {
So(transformed.ReadAll(), ShouldResemble, []stream.T{1, 2})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
transformer := transformers.TakeFirst(1)
transformer.Attach(context)
next := transformer.Transform(in)
Convey("Then no item is sent to the next stage", func() {
So(next.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestLastItemCollector(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the collector consumer", func() {
var number int
consumer := consumers.LastItemCollector(&number)
consumer.Attach(context)
consumer.Consume(in)
Convey("Then data is collected out of the stream", func() {
So(number, ShouldResemble, 2)
_, opened := <-in
So(opened, ShouldBeFalse)
})
})
Convey("When I apply the collector consuming data into a non pointer", func() {
var number int
collect := func() {
consumers.LastItemCollector(number)
}
Convey("Then it panics", func() {
So(collect, ShouldPanicWith, consumers.ErrNoSuchPointer)
})
})
})
})
}
func TestDrainer(t *testing.T) {
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(2)
out <- 1
out <- 2
close(out)
Convey("When I apply the drainer consumer", func() {
consumer := consumers.Drainer()
consumer.Attach(context)
consumer.Consume(in)
Convey("Then the stream is drained", func() {
data, opened := <-in
So(data, ShouldBeNil)
So(opened, ShouldBeFalse)
})
})
})
})
}
func TestIfDispatcher(t *testing.T) {
evens := func(d stream.T) bool { return d.(int)%2 == 0 }
Convey("Given I have a context", t, func() {
context := rivers.NewContext()
Convey("And a stream of data", func() {
in, out := stream.New(3)
out <- 2
out <- 3
out <- 4
close(out)
Convey("When I apply an if dispatcher", func() {
evensIn, evensOut := stream.New(3)
sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)
Convey("Then items matching the condition are dispatched to the corresponding stream", func() {
data := evensIn.ReadAll()
So(data, should.Contain, 2)
So(data, should.Contain, 4)
Convey("And items not matching the condition are dispatched to the sink stream", func() {
So(sink.ReadAll(), ShouldResemble, []stream.T{3})
})
})
})
Convey("When I apply an always dispatcher", func() {
streamIn1, streamOut1 := stream.New(3)
streamIn2, streamOut2 := stream.New(3)
sink := dispatchers.New(context).Always().Dispatch(in, streamOut1, streamOut2)
Convey("Then all items are dispatched to the corresponding streams", func() {
streamIn1Items := streamIn1.ReadAll()
streamIn2Items := streamIn2.ReadAll()
So(streamIn1Items, should.Contain, 2)
So(streamIn1Items, should.Contain, 3)
So(streamIn1Items, should.Contain, 4)
So(streamIn2Items, should.Contain, 2)
So(streamIn2Items, should.Contain, 3)
So(streamIn2Items, should.Contain, 4)
Convey("And no item is dispatched to the sink stream", func() {
So(sink.ReadAll(), ShouldBeEmpty)
})
})
})
Convey("When I close the context", func() {
context.Close(stream.Done)
Convey("And I apply the transformer to the stream", func() {
evensIn, evensOut := stream.New(3)
sink := dispatchers.New(context).If(evens).Dispatch(in, evensOut)
Convey("Then no item is sent to the next stage", func() {
So(evensIn.ReadAll(), ShouldBeEmpty)
So(sink.ReadAll(), ShouldBeEmpty)
})
})
})
})
})
}
func TestLoadBatchFromCache(t *testing.T) {
gaeCtx, _ := aetest.NewContext(nil)
defer gaeCtx.Close()
Convey("Given I have a load batch from cache transformer", t, func() {
riversCtx := rivers.NewContext()
loadBatchProcessor := appx.NewStep(riversCtx).LoadBatchFromCache(gaeCtx)
Convey("And I have a few entities in the cache", func() {
user1 := NewUser(User{
Name: "Borges",
Email: "*****@*****.**",
SSN: "123123123",
})
user2 := NewUser(User{
Name: "Diego",
Email: "*****@*****.**",
SSN: "321321",
})
appx.NewKeyResolver(gaeCtx).Resolve(user1)
appx.NewKeyResolver(gaeCtx).Resolve(user2)
memcache.JSON.Set(gaeCtx, &memcache.Item{
Key: user1.CacheID(),
Object: appx.CachedEntity{
Entity: user1,
Key: user1.Key(),
},
})
memcache.JSON.Set(gaeCtx, &memcache.Item{
Key: user2.CacheID(),
Object: appx.CachedEntity{
Entity: user2,
Key: user2.Key(),
},
})
Convey("When I transform the incoming batch", func() {
notCachedUser := NewUser(User{
Name: "not cached",
SSN: "notcached",
})
userFromCache1 := NewUser(User{Name: user1.Name})
userFromCache2 := NewUser(User{Name: user2.Name})
batchItems := make(map[string]*appx.CachedEntity)
batchItems[user1.CacheID()] = &appx.CachedEntity{
Entity: userFromCache1,
}
batchItems[user2.CacheID()] = &appx.CachedEntity{
Entity: userFromCache2,
}
batchItems[notCachedUser.CacheID()] = &appx.CachedEntity{
Entity: notCachedUser,
}
batch := &appx.MemcacheLoadBatch{
Keys: []string{user1.CacheID(), user2.CacheID()},
Items: batchItems,
}
in, out := stream.New(1)
loadBatchProcessor(batch, stream.NewEmitter(rivers.NewContext(), out))
close(out)
Convey("Then cache misses are sent downstream", func() {
So(in.ReadAll(), ShouldResemble, []stream.T{notCachedUser})
Convey("And entities are loaded from cache", func() {
So(userFromCache1, ShouldResemble, user1)
So(userFromCache2, ShouldResemble, user2)
})
})
})
})
})
}
func TestQueryEntityFromDatastore(t *testing.T) {
gaeCtx, _ := aetest.NewContext(nil)
defer gaeCtx.Close()
user := &User{
Name: "Borges",
Email: "*****@*****.**",
SSN: "123123123",
keySpec: &appx.KeySpec{
Kind: "Users",
StringID: "borges",
},
}
parentKey := datastore.NewKey(gaeCtx, "Parent", "parent id", 0, nil)
user.SetParentKey(parentKey)
Convey("Given I have a query entity from datastore transformer", t, func() {
riversCtx := rivers.NewContext()
queryProcessor := appx.NewStep(riversCtx).QueryEntityFromDatastore(gaeCtx)
Convey("When I transform the inbound stream with non existent entity", func() {
nonExistentUser := &User{
Email: "*****@*****.**",
keySpec: &appx.KeySpec{
Kind: "Users",
},
}
runQuery := func() {
queryProcessor(nonExistentUser)
}
Convey("Then query processor panics", func() {
So(runQuery, ShouldPanic)
})
})
Convey("And I have an entity in datastore", func() {
err := appx.NewKeyResolver(gaeCtx).Resolve(user)
So(err, ShouldBeNil)
_, err = datastore.Put(gaeCtx, user.Key(), user)
So(err, ShouldBeNil)
// Give datastore some time so that the created entity is available to be queried
time.Sleep(200 * time.Millisecond)
Convey("When I transform the inbound stream", func() {
userFromDatastore := &User{
Email: "*****@*****.**",
keySpec: &appx.KeySpec{
Kind: "Users",
},
}
queryProcessor(userFromDatastore)
Convey("And queryable entities are loaded from datastore", func() {
So(userFromDatastore.Name, ShouldEqual, user.Name)
So(userFromDatastore.Email, ShouldEqual, user.Email)
So(userFromDatastore.Key(), ShouldResemble, user.Key())
So(userFromDatastore.ParentKey(), ShouldResemble, user.ParentKey())
})
})
})
})
}
