示例#1
0
func TestBinaryOp_Params(t *testing.T) {
	o := NewBinaryOp(context.Background())
	op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
		return nil
	})
	in := make(chan interface{})

	o.SetOperation(op)
	if o.op == nil {
		t.Fatal("process Elem not set")
	}

	o.SetConcurrency(4)
	if o.concurrency != 4 {
		t.Fatal("Concurrency not being set")
	}

	o.SetInput(in)
	if o.input == nil {
		t.Fatal("Input not being set")
	}

	if o.GetOutput == nil {
		t.Fatal("Output not set")
	}
}
示例#2
0
func BenchmarkBinaryOp_Exec(b *testing.B) {
	ctx := context.Background()
	o := NewBinaryOp(ctx)
	N := b.N

	chanSize := func() int {
		if N == 1 {
			return N
		}
		return int(float64(0.5) * float64(N))
	}()

	in := make(chan interface{}, chanSize)
	o.SetInput(in)
	o.SetInitialState(0)
	go func() {
		for i := 0; i < N; i++ {
			in <- len(testutil.GenWord())
		}
		close(in)
	}()

	op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
		val0 := op1.(int)
		val1 := op2.(int)
		return val0 + val1
	})
	o.SetOperation(op)

	// process output

	if err := o.Exec(); err != nil {
		b.Fatal("Error during execution:", err)
	}

	select {
	case out := <-o.GetOutput():
		val := out.(int)
		if val == 0 {
			b.Fatal("Numbers did not get added")
		}
	case <-time.After(time.Second * 60):
		b.Fatal("Took too long")
	}
}
示例#3
0
// Reduce accumulates and reduce a stream of elements into a single value
func (s *Stream) Reduce(f interface{}) *Stream {
	fntype := reflect.TypeOf(f)
	if err := s.isBinaryFuncForm(fntype); err != nil {
		panic(fmt.Sprintf("Op Reduce() failed: %s", err))
	}

	fnval := reflect.ValueOf(f)

	op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
		arg0 := reflect.ValueOf(op1)
		arg1, arg1Type := reflect.ValueOf(op2), reflect.TypeOf(op2)
		if op1 == nil {
			arg0 = reflect.Zero(arg1Type)
		}
		result := fnval.Call([]reflect.Value{arg0, arg1})[0]
		return result.Interface()
	})

	return s.Accumulate(op)
}
示例#4
0
func TestBinaryOp_Exec(t *testing.T) {
	ctx, _ := context.WithCancel(context.Background())
	o := NewBinaryOp(ctx)

	o.SetInitialState(0)
	op := api.BinFunc(func(ctx context.Context, op1, op2 interface{}) interface{} {
		init := op1.(int)
		items := op2.([]int)
		for _, item := range items {
			init += item
		}
		return init
	})
	o.SetOperation(op)

	in := make(chan interface{})
	go func() {
		in <- []int{1}
		in <- []int{1, 2}
		in <- []int{1, 2, 3}
		close(in)
	}()
	o.SetInput(in)

	if err := o.Exec(); err != nil {
		t.Fatal(err)
	}

	select {
	case out := <-o.GetOutput():
		val := out.(int)
		if val != 10 {
			t.Fatal("Values not adding up to expected 10")
		}
	case <-time.After(50 * time.Millisecond):
		t.Fatal("Took too long...")
	}
}
示例#5
0
// groupByInt expects incoming data as Pair, []slice, or [n]array.
// It creates the reduction operation and stores incoming data in a map.
func (s *Stream) groupByInt(i int64) api.BinFunc {
	op := api.BinFunc(func(ctx context.Context, op0, op1 interface{}) interface{} {
		stateType := reflect.TypeOf(op0)
		if stateType.Kind() != reflect.Map {
			panic("GroupBy expects a map[keytype][]slice for internal storage")
		}
		stateMap := reflect.ValueOf(op0)

		// save data according to type
		dataType := reflect.TypeOf(op1)
		dataVal := reflect.ValueOf(op1)
		switch dataType.Kind() {
		case reflect.Slice, reflect.Array:
			// build stateMap[key][]slice dynamically. Add item to slice.
			key := dataVal.Index(int(i))
			if key.IsValid() {
				slice := stateMap.MapIndex(key)
				if !slice.IsValid() {
					slice = reflect.MakeSlice(stateType.Elem(), 0, 0)
					stateMap.SetMapIndex(key, slice)
				}

				// copy value to group in new slice
				for j := 0; j < dataVal.Len(); j++ {
					if j != int(i) {
						slice = reflect.Append(slice, dataVal.Index(j))
					}
				}
				stateMap.SetMapIndex(key, slice)
			}
		default: // ignore anything else
		}

		return stateMap.Interface()
	})

	return op
}
示例#6
0
// groupByName returns a binary function that groups streaming item
// by a struct.name or a map[name].  If the item is not a struct or map,
// it is ignored.
func (s *Stream) groupByName(name string) api.BinFunc {
	op := api.BinFunc(func(ctx context.Context, op0, op1 interface{}) interface{} {
		stateType := reflect.TypeOf(op0)
		if stateType.Kind() != reflect.Map {
			panic("GroupBy expects a map[keytype][]slice for internal storage")
		}
		stateMap := reflect.ValueOf(op0)

		// stream item data type and value
		dataType := reflect.TypeOf(op1)
		dataVal := reflect.ValueOf(op1)
		key := dataVal.FieldByName(name)

		if key.IsValid() {
			// ensure state map[K][]slice is ready
			slice := stateMap.MapIndex(key)
			if !slice.IsValid() {
				slice = reflect.MakeSlice(stateType.Elem(), 0, 0)
				stateMap.SetMapIndex(key, slice)
			}

			switch dataType.Kind() {
			// append struct.name = V to state map[name][]slice{V}
			case reflect.Struct:
				slice = reflect.Append(slice, dataVal)
				stateMap.SetMapIndex(key, slice)
			default:
			}
		}

		return stateMap.Interface()

	})

	return op
}