func TestStream_Open_WithOp(t *testing.T) {
src := newStrSrc([]string{"HELLO", "WORLD", "HOW", "ARE", "YOU"})
snk := newStrSink()
op1 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
str := data.(string)
return len(str)
})
var m sync.RWMutex
runeCount := 0
op2 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
length := data.(int)
m.Lock()
runeCount += length
m.Unlock()
return nil
})
strm := New().From(src).Transform(op1).Transform(op2).To(snk)
select {
case err := <-strm.Open():
if err != nil {
t.Fatal(err)
}
case <-time.After(50 * time.Millisecond):
t.Fatal("Waited too long ...")
}
m.RLock()
if runeCount != 19 {
t.Fatal("Data not streaming, runeCount 19, got ", runeCount)
}
m.RUnlock()
}
func TestUnaryOp_Params(t *testing.T) {
o := NewUnaryOp(context.Background())
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
return nil
})
in := make(chan interface{})
o.SetOperation(op)
if o.op == nil {
t.Fatal("process Elem not set")
}
o.SetConcurrency(4)
if o.concurrency != 4 {
t.Fatal("Concurrency not being set")
}
o.SetInput(in)
if o.input == nil {
t.Fatal("Input not being set")
}
if o.GetOutput == nil {
t.Fatal("Output not set")
}
}
func BenchmarkUnaryOp_Exec(b *testing.B) {
ctx := context.Background()
o := NewUnaryOp(ctx)
N := b.N
chanSize := func() int {
if N == 1 {
return N
}
return int(float64(0.5) * float64(N))
}()
in := make(chan interface{}, chanSize)
o.SetInput(in)
go func() {
for i := 0; i < N; i++ {
in <- testutil.GenWord()
}
close(in)
}()
counter := 0
var m sync.RWMutex
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
m.Lock()
counter++
m.Unlock()
return data
})
o.SetOperation(op)
// process output
done := make(chan struct{})
go func() {
defer close(done)
for _ = range o.GetOutput() {
}
}()
if err := o.Exec(); err != nil {
b.Fatal("Error during execution:", err)
}
select {
case <-done:
case <-time.After(time.Second * 60):
b.Fatal("Took too long")
}
m.RLock()
b.Logf("Input %d, counted %d", N, counter)
if counter != N {
b.Fatalf("Expected %d items processed, got %d", N, counter)
}
m.RUnlock()
}
func TestStream_InitGraph(t *testing.T) {
src := newStrSrc([]string{"Hello", "World"})
snk := newStrSink()
op1 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
return nil
})
op2 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
return nil
})
strm := New().From(src).To(snk)
if err := strm.initGraph(); err != nil {
t.Fatal(err)
}
if src.GetOutput() != snk.input {
t.Fatal("Source not link to sink when no ops are present")
}
strm = New().From(src).Transform(op1).Transform(op2).To(snk)
if err := strm.initGraph(); err != nil {
t.Fatal(err)
}
if len(strm.ops) != 2 {
t.Fatal("Not adding operations to stream")
}
if src.GetOutput() == snk.input {
t.Fatal("Graph invalid, source skipping ops, linked to sink!")
}
if strm.ops[1].GetOutput() != snk.input {
t.Fatal("Sink not linked to last element in graph")
}
}
// Process is used to express general processing operation of incoming stream
// elements. Function must be of the form "func(input)output", anything else
// will cause painic.
func (s *Stream) Process(f interface{}) *Stream {
fntype := reflect.TypeOf(f)
if err := s.isUnaryFuncForm(fntype); err != nil {
panic(fmt.Sprintf("Op Process() failed: %s", err))
}
fnval := reflect.ValueOf(f)
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
arg0 := reflect.ValueOf(data)
result := fnval.Call([]reflect.Value{arg0})[0]
return result.Interface()
})
return s.Transform(op)
}
func TestUnaryOp_Exec(t *testing.T) {
ctx, _ := context.WithCancel(context.Background())
o := NewUnaryOp(ctx)
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
values := data.([]string)
t.Logf("Processing data %v, sending %d", values, len(values))
return len(values)
})
o.SetOperation(op)
in := make(chan interface{})
go func() {
in <- []string{"A", "B", "C"}
in <- []string{"D", "E"}
in <- []string{"G"}
close(in)
}()
o.SetInput(in)
wait := make(chan struct{})
go func() {
defer close(wait)
for data := range o.GetOutput() {
val, ok := data.(int)
t.Logf("Got value %v", val)
if !ok {
t.Fatalf("Expecting type int, got %T, value %v", val, val)
}
if val != 3 && val != 2 && val != 1 {
t.Fatalf("Expecting values 3, 2, or 1, but got %d", val)
}
}
}()
if err := o.Exec(); err != nil {
t.Fatal(err)
}
select {
case <-wait:
case <-time.After(50 * time.Millisecond):
t.Fatal("Took too long...")
}
}
// FlatMap similar to Map, however, expected to return a slice of values
// to downstream operators. The operator will flatten the slice and emit
// individually onto the stream. The FlatMap function must have the form
// "func(intput)[]output", anything else will be rejected
func (s *Stream) FlatMap(f interface{}) *Stream {
fntype := reflect.TypeOf(f)
if err := s.isUnaryFuncForm(fntype); err != nil {
panic(fmt.Sprintf("Op FlatMap() failed: %s", err))
}
if fntype.Out(0).Kind() != reflect.Slice {
panic(fmt.Sprintf("Op FlatMap() expects to return a slice of values"))
}
fnval := reflect.ValueOf(f)
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
arg0 := reflect.ValueOf(data)
result := fnval.Call([]reflect.Value{arg0})[0]
return result.Interface()
})
s.Transform(op) // add flatmap as unary op
s.ReStream() // add streamop to unpack flatmap result
return s
}
func TestStream_BuilderMethods(t *testing.T) {
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
return nil
})
st := New()
st.
From(newStrSrc([]string{"Hello", "World", "!!"})).
To(newStrSink()).
Transform(op)
if st.source == nil {
t.Fatal("From() not setting source")
}
if st.sink == nil {
t.Fatal("To() not setting sink")
}
if len(st.ops) != 1 {
t.Fatal("Operation not added to ops slice")
}
}
// Filter takes a predicate func that filters the stream.
// Function must be of form "func(input)bool", anything else causes panic.
// If func returns true, current item continues downstream.
func (s *Stream) Filter(f interface{}) *Stream {
fntype := reflect.TypeOf(f)
if err := s.isUnaryFuncForm(fntype); err != nil {
panic(fmt.Sprintf("Op Filter() failed :%s", err))
}
// ensure bool ret type
if fntype.Out(0).Kind() != reflect.Bool {
panic("Op Filter() must return a bool")
}
fnval := reflect.ValueOf(f)
op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
arg0 := reflect.ValueOf(data)
result := fnval.Call([]reflect.Value{arg0})[0]
predicate := result.Bool()
if !predicate {
return nil
}
return data
})
return s.Transform(op)
}