func TestBinaryOp_Params(t *testing.T) {
o := NewBinaryOp(context.Background())
op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
return nil
})
in := make(chan interface{})
o.SetOperation(op)
if o.op == nil {
t.Fatal("process Elem not set")
}
o.SetConcurrency(4)
if o.concurrency != 4 {
t.Fatal("Concurrency not being set")
}
o.SetInput(in)
if o.input == nil {
t.Fatal("Input not being set")
}
if o.GetOutput == nil {
t.Fatal("Output not set")
}
}
func BenchmarkBinaryOp_Exec(b *testing.B) {
ctx := context.Background()
o := NewBinaryOp(ctx)
N := b.N
chanSize := func() int {
if N == 1 {
return N
}
return int(float64(0.5) * float64(N))
}()
in := make(chan interface{}, chanSize)
o.SetInput(in)
o.SetInitialState(0)
go func() {
for i := 0; i < N; i++ {
in <- len(testutil.GenWord())
}
close(in)
}()
op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
val0 := op1.(int)
val1 := op2.(int)
return val0 + val1
})
o.SetOperation(op)
// process output
if err := o.Exec(); err != nil {
b.Fatal("Error during execution:", err)
}
select {
case out := <-o.GetOutput():
val := out.(int)
if val == 0 {
b.Fatal("Numbers did not get added")
}
case <-time.After(time.Second * 60):
b.Fatal("Took too long")
}
}
// Reduce accumulates and reduce a stream of elements into a single value
func (s *Stream) Reduce(f interface{}) *Stream {
fntype := reflect.TypeOf(f)
if err := s.isBinaryFuncForm(fntype); err != nil {
panic(fmt.Sprintf("Op Reduce() failed: %s", err))
}
fnval := reflect.ValueOf(f)
op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
arg0 := reflect.ValueOf(op1)
arg1, arg1Type := reflect.ValueOf(op2), reflect.TypeOf(op2)
if op1 == nil {
arg0 = reflect.Zero(arg1Type)
}
result := fnval.Call([]reflect.Value{arg0, arg1})[0]
return result.Interface()
})
return s.Accumulate(op)
}
func TestBinaryOp_Exec(t *testing.T) {
ctx, _ := context.WithCancel(context.Background())
o := NewBinaryOp(ctx)
o.SetInitialState(0)
op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
init := op1.(int)
items := op2.([]int)
for _, item := range items {
init += item
}
return init
})
o.SetOperation(op)
in := make(chan interface{})
go func() {
in <- []int{1}
in <- []int{1, 2}
in <- []int{1, 2, 3}
close(in)
}()
o.SetInput(in)
if err := o.Exec(); err != nil {
t.Fatal(err)
}
select {
case out := <-o.GetOutput():
val := out.(int)
if val != 10 {
t.Fatal("Values not adding up to expected 10")
}
case <-time.After(50 * time.Millisecond):
t.Fatal("Took too long...")
}
}
// groupByInt expects incoming data as Pair, []slice, or [n]array.
// It creates the reduction operation and stores incoming data in a map.
func (s *Stream) groupByInt(i int64) api.BinFunc {
op := api.BinFunc(func(ctx context.Context, op0, op1 interface{}) interface{} {
stateType := reflect.TypeOf(op0)
if stateType.Kind() != reflect.Map {
panic("GroupBy expects a map[keytype][]slice for internal storage")
}
stateMap := reflect.ValueOf(op0)
// save data according to type
dataType := reflect.TypeOf(op1)
dataVal := reflect.ValueOf(op1)
switch dataType.Kind() {
case reflect.Slice, reflect.Array:
// build stateMap[key][]slice dynamically. Add item to slice.
key := dataVal.Index(int(i))
if key.IsValid() {
slice := stateMap.MapIndex(key)
if !slice.IsValid() {
slice = reflect.MakeSlice(stateType.Elem(), 0, 0)
stateMap.SetMapIndex(key, slice)
}
// copy value to group in new slice
for j := 0; j < dataVal.Len(); j++ {
if j != int(i) {
slice = reflect.Append(slice, dataVal.Index(j))
}
}
stateMap.SetMapIndex(key, slice)
}
default: // ignore anything else
}
return stateMap.Interface()
})
return op
}
// groupByName returns a binary function that groups streaming item
// by a struct.name or a map[name]. If the item is not a struct or map,
// it is ignored.
func (s *Stream) groupByName(name string) api.BinFunc {
op := api.BinFunc(func(ctx context.Context, op0, op1 interface{}) interface{} {
stateType := reflect.TypeOf(op0)
if stateType.Kind() != reflect.Map {
panic("GroupBy expects a map[keytype][]slice for internal storage")
}
stateMap := reflect.ValueOf(op0)
// stream item data type and value
dataType := reflect.TypeOf(op1)
dataVal := reflect.ValueOf(op1)
key := dataVal.FieldByName(name)
if key.IsValid() {
// ensure state map[K][]slice is ready
slice := stateMap.MapIndex(key)
if !slice.IsValid() {
slice = reflect.MakeSlice(stateType.Elem(), 0, 0)
stateMap.SetMapIndex(key, slice)
}
switch dataType.Kind() {
// append struct.name = V to state map[name][]slice{V}
case reflect.Struct:
slice = reflect.Append(slice, dataVal)
stateMap.SetMapIndex(key, slice)
default:
}
}
return stateMap.Interface()
})
return op
}