func (datastore *Datastore) Load(entities ...Entity) error { pipeline := rivers.FromSlice(entities) step := NewStep(pipeline.Context) cacheableEntities, nonCacheableEntities := pipeline. Each(step.ResolveEntityKeySilently(datastore.context)). Partition(step.CacheableEntitiesWithCacheKey) cacheMisses, cacheMissesToBeCached := cacheableEntities. BatchBy(step.MemcacheLoadBatchOf(1000)). OnData(step.LoadBatchFromCache(datastore.context)). Split() entitiesWithKeys, entitiesMissingKeys := pipeline.Merge(nonCacheableEntities, cacheMisses). Partition(step.ResolvedKeys) notLoadedEntities := entitiesWithKeys. BatchBy(step.DatastoreBatchOf(1000)). Each(step.LoadBatchFromDatastore(datastore.context)) notQueriedEntities := entitiesMissingKeys. Each(step.QueryEntityFromDatastore(datastore.context)) pipeline.Merge(notLoadedEntities, notQueriedEntities).Drain() return cacheMissesToBeCached. Filter(step.EntitiesWithNonEmptyCacheIDs). BatchBy(step.MemcacheSaveBatchOf(1000)). Each(step.SaveMemcacheBatch(datastore.context)). Drain() }
func (datastore *Datastore) Delete(entities ...Entity) error { pipeline := rivers.FromSlice(entities) step := NewStep(pipeline.Context) deleteFromCache, deleteFromDatastore := pipeline. Each(step.ResolveEntityKey(datastore.context)). Split() cacheStream := deleteFromCache. Filter(step.CacheableEntitiesWithCacheKey). BatchBy(step.MemcacheDeleteBatchOf(500)). Each(step.DeleteBatchFromCache(datastore.context)) datastoreStream := deleteFromDatastore. BatchBy(step.DatastoreBatchOf(500)). Each(step.DeleteBatchFromDatastore(datastore.context)) return pipeline.Merge(cacheStream, datastoreStream).Drain() }
func (datastore *Datastore) Save(entities ...Entity) error { pipeline := rivers.FromSlice(entities) step := NewStep(pipeline.Context) entitiesToBeCached, entitiesToBeSavedInDatastore := pipeline. Each(step.ResolveEntityKey(datastore.context)). Split() cacheStream := entitiesToBeCached. Filter(step.CacheableEntitiesWithCacheKey). BatchBy(step.MemcacheSaveBatchOf(500)). Each(step.SaveMemcacheBatch(datastore.context)) datastoreStream := entitiesToBeSavedInDatastore. BatchBy(step.DatastoreBatchOf(500)). Each(step.SaveDatastoreBatch(datastore.context)) return pipeline.Merge(cacheStream, datastoreStream).Drain() }
func TestRiversAPI(t *testing.T) { evensOnly := func(data stream.T) bool { return data.(int)%2 == 0 } sum := func(a, b stream.T) stream.T { return a.(int) + b.(int) } add := func(n int) stream.MapFn { return func(data stream.T) stream.T { return data.(int) + n } } concat := func(c string) stream.MapFn { return func(data stream.T) stream.T { return data.(string) + c } } addOrAppend := func(n int, c string) stream.MapFn { return func(data stream.T) stream.T { if num, ok := data.(int); ok { return num + n } if letter, ok := data.(string); ok { return letter + "_" } return data } } alphabeticOrder := func(a, b stream.T) bool { return a.(string) < b.(string) } Convey("rivers API", t, func() { Convey("From Range -> Filter -> Map -> Reduce -> Each", func() { data, _ := rivers.FromRange(1, 5). Filter(evensOnly). Map(add(1)). Reduce(0, sum). Collect() So(data, ShouldResemble, []stream.T{8}) }) Convey("From Data -> Flatten -> Map -> Sort By", func() { items, err := rivers.FromData([]stream.T{"a", "c"}, "b", []stream.T{"d", "e"}). Flatten(). Map(concat("_")). SortBy(alphabeticOrder) So(err, ShouldBeNil) So(items, ShouldResemble, []stream.T{"a_", "b_", "c_", "d_", "e_"}) }) Convey("From Data -> FlatMap", func() { data, _ := rivers.FromRange(1, 3). FlatMap(func(data stream.T) stream.T { return []stream.T{data, data.(int) + 1} }). Collect() So(data, ShouldResemble, []stream.T{1, 2, 2, 3, 3, 4}) }) Convey("From Slice -> Dispatch If -> Map", func() { in, out := stream.New(2) notDispatched, _ := rivers.FromSlice([]stream.T{1, 2, 3, 4, 5}). DispatchIf(evensOnly, out). Map(add(2)). Collect() data := in.ReadAll() So(data, ShouldContain, 2) So(data, ShouldContain, 4) So(notDispatched, ShouldContain, 3) So(notDispatched, ShouldContain, 5) So(notDispatched, ShouldContain, 7) }) Convey("Zip -> Map", func() { numbers := rivers.FromData(1, 2, 3, 4) letters := rivers.FromData("a", "b", "c") combined, _ := numbers.Zip(letters).Map(addOrAppend(1, "_")).Collect() So(combined, ShouldResemble, []stream.T{2, "a_", 3, "b_", 4, "c_", 5}) }) Convey("Zip By -> Filter -> Collect", func() { numbers := rivers.FromData(1, 2, 3, 4) moreNumbers := rivers.FromData(4, 4, 1) combined, err := numbers.ZipBy(sum, moreNumbers).Filter(evensOnly).Collect() So(err, ShouldBeNil) So(combined, ShouldResemble, []stream.T{6, 4, 4}) }) Convey("Merge -> Map", func() { numbers := rivers.FromData(1, 2) moreNumbers := rivers.FromData(3, 4) combined, _ := numbers.Merge(moreNumbers).Collect() So(len(combined), ShouldEqual, 4) So(combined, ShouldContain, 1) So(combined, ShouldContain, 2) So(combined, ShouldContain, 3) So(combined, ShouldContain, 4) }) Convey("From Data -> Drain", func() { numbers := rivers.FromData(1, 2, 3, 4) numbers.Drain() data, opened := <-numbers.Stream So(data, ShouldBeNil) So(opened, ShouldBeFalse) }) Convey("From Range -> Partition", func() { evensStage, oddsStage := rivers.FromRange(1, 4).Partition(evensOnly) evens, _ := evensStage.Collect() odds, _ := oddsStage.Collect() So(evens, ShouldContain, 2) So(evens, ShouldContain, 4) So(odds, ShouldContain, 1) So(odds, ShouldContain, 3) }) Convey("From Range -> Slipt", func() { lhs, rhs := rivers.FromRange(1, 2).Split() lhsData := lhs.Stream.ReadAll() rhsData := rhs.Stream.ReadAll() So(lhsData, ShouldContain, 1) So(lhsData, ShouldContain, 2) So(rhsData, ShouldContain, 1) So(rhsData, ShouldContain, 2) }) Convey("From Range -> Slipt N", func() { pipelines := rivers.FromRange(1, 2).SplitN(3) data0 := pipelines[0].Stream.ReadAll() data1 := pipelines[1].Stream.ReadAll() data2 := pipelines[2].Stream.ReadAll() So(data0, ShouldContain, 1) So(data0, ShouldContain, 2) So(data1, ShouldContain, 1) So(data1, ShouldContain, 2) So(data2, ShouldContain, 1) So(data2, ShouldContain, 2) }) Convey("From Range -> OnData", func() { pipeline := rivers.FromRange(1, 4).OnData(func(data stream.T, emitter stream.Emitter) { if data.(int)%2 == 0 { emitter.Emit(data) } }) So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4}) }) Convey("From Range -> TakeFirst N -> Collect", func() { taken, _ := rivers.FromRange(1, 4).TakeFirst(2).Collect() So(taken, ShouldResemble, []stream.T{1, 2}) }) Convey("From Range -> Take", func() { pipeline := rivers.FromRange(1, 4).Take(evensOnly) So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{2, 4}) }) Convey("From Range -> Drop", func() { pipeline := rivers.FromRange(1, 4).Drop(evensOnly) So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{1, 3}) }) Convey("From Range -> Drop First 2", func() { pipeline := rivers.FromRange(1, 5).DropFirst(2) So(pipeline.Stream.ReadAll(), ShouldResemble, []stream.T{3, 4, 5}) }) Convey("From Range -> Collect", func() { data, err := rivers.FromRange(1, 4).Collect() So(err, ShouldBeNil) So(data, ShouldResemble, []stream.T{1, 2, 3, 4}) }) Convey("From Range -> CollectFirst", func() { data, err := rivers.FromRange(1, 4).CollectFirst() So(err, ShouldBeNil) So(data, ShouldEqual, 1) }) Convey("From Range -> CollectFirstAs", func() { var data int err := rivers.FromRange(1, 4).CollectFirstAs(&data) So(err, ShouldBeNil) So(data, ShouldEqual, 1) }) Convey("From Range -> CollectLast", func() { data, err := rivers.FromRange(1, 4).CollectLast() So(err, ShouldBeNil) So(data, ShouldEqual, 4) }) Convey("From Range -> CollectLastAs", func() { var data int err := rivers.FromRange(1, 4).CollectLastAs(&data) So(err, ShouldBeNil) So(data, ShouldEqual, 4) }) Convey("From Range -> CollectAs", func() { var numbers []int err := rivers.FromRange(1, 4).CollectAs(&numbers) So(err, ShouldBeNil) So(numbers, ShouldResemble, []int{1, 2, 3, 4}) }) Convey("From Data -> Map From Struct To JSON", func() { type Account struct{ Name string } items := rivers.FromData(Account{"Diego"}).Map(from.StructToJSON).Stream.ReadAll() So(items, ShouldResemble, []stream.T{[]byte(`{"Name":"Diego"}`)}) }) Convey("From Data -> Map From JSON To Struct", func() { type Account struct{ Name string } items := rivers.FromData([]byte(`{"Name":"Diego"}`)).Map(from.JSONToStruct(Account{})).Stream.ReadAll() So(items, ShouldResemble, []stream.T{Account{"Diego"}}) }) Convey("From Range -> Collect By", func() { items := []stream.T{} err := rivers.FromRange(1, 5).CollectBy(func(data stream.T) { items = append(items, data) }) So(err, ShouldBeNil) So(items, ShouldResemble, []stream.T{1, 2, 3, 4, 5}) }) Convey("From Range -> Find", func() { data, err := rivers.FromRange(1, 5).Find(2).Collect() So(err, ShouldBeNil) So(data, ShouldResemble, []stream.T{2}) }) Convey("From Range -> Find By", func() { data, err := rivers.FromRange(1, 5).FindBy(func(subject stream.T) bool { return subject == 2 }).Collect() So(err, ShouldBeNil) So(data, ShouldResemble, []stream.T{2}) }) Convey("From Range -> Parallel -> Each", func() { start := time.Now() err := rivers.FromRange(1, 10).Parallel().Each(func(data stream.T) { time.Sleep(500 * time.Millisecond) }).Drain() end := time.Since(start) So(err, ShouldBeNil) So(end.Seconds(), ShouldBeLessThanOrEqualTo, 1) }) Convey("From Slow Producer -> Find", func() { slowProducer := &producers.Observable{ Capacity: 2, Emit: func(emitter stream.Emitter) { for i := 0; i < 5; i++ { emitter.Emit(i) time.Sleep(300 * time.Millisecond) } }, } items, err := rivers.From(slowProducer).Find(2).Collect() So(err, ShouldBeNil) So(items, ShouldResemble, []stream.T{2}) }) Convey("Producer times out", func() { slowProducer := &producers.Observable{ Capacity: 2, Emit: func(emitter stream.Emitter) { for i := 0; i < 5; i++ { emitter.Emit(i) time.Sleep(2 * time.Second) } }, } items, err := rivers.From(slowProducer).Deadline(300 * time.Millisecond).Find(2).Collect() So(err, ShouldEqual, stream.Timeout) So(items, ShouldBeNil) }) Convey("Transformer times out", func() { slowTransformer := &transformers.Observer{ OnNext: func(data stream.T, emitter stream.Emitter) error { time.Sleep(2 * time.Second) emitter.Emit(data) return nil }, } items, err := rivers.FromRange(1, 3).Deadline(300 * time.Millisecond).Apply(slowTransformer).Collect() So(err, ShouldEqual, stream.Timeout) So(items, ShouldBeNil) }) Convey("From Range -> Group By", func() { evensAndOdds := func(data stream.T) (key stream.T) { if data.(int)%2 == 0 { return "evens" } return "odds" } groups, err := rivers.FromRange(1, 5).GroupBy(evensAndOdds) So(err, ShouldBeNil) So(groups.Empty(), ShouldBeFalse) So(groups.HasGroup("evens"), ShouldBeTrue) So(groups.HasGroup("odds"), ShouldBeTrue) So(groups.HasGroup("invalid"), ShouldBeFalse) So(groups.HasItem(2), ShouldBeTrue) So(groups.HasItem(6), ShouldBeFalse) So(groups, ShouldResemble, stream.Groups{ "evens": []stream.T{2, 4}, "odds": []stream.T{1, 3, 5}, }) }) Convey("From Reader -> Map -> Filter -> Collect", func() { toString := func(data stream.T) stream.T { return string(data.(byte)) } dashes := func(data stream.T) bool { return data == "-" } items, err := rivers.FromReader(bytes.NewReader([]byte("abcd"))).Map(toString).Drop(dashes).Collect() So(err, ShouldBeNil) So(items, ShouldResemble, []stream.T{"a", "b", "c", "d"}) }) Convey("From Range -> Count", func() { count, err := rivers.FromRange(1, 5).Count() So(err, ShouldBeNil) So(count, ShouldEqual, 5) }) }) }
func From(slice []string) *Pipeline { return &Pipeline{rivers.FromSlice(slice)} }