Exemple #1
0
func TestStream_Open_WithOp(t *testing.T) {
	src := newStrSrc([]string{"HELLO", "WORLD", "HOW", "ARE", "YOU"})
	snk := newStrSink()
	op1 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		str := data.(string)
		return len(str)
	})

	var m sync.RWMutex
	runeCount := 0
	op2 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		length := data.(int)
		m.Lock()
		runeCount += length
		m.Unlock()
		return nil
	})

	strm := New().From(src).Transform(op1).Transform(op2).To(snk)
	select {
	case err := <-strm.Open():
		if err != nil {
			t.Fatal(err)
		}
	case <-time.After(50 * time.Millisecond):
		t.Fatal("Waited too long ...")
	}
	m.RLock()
	if runeCount != 19 {
		t.Fatal("Data not streaming, runeCount 19, got ", runeCount)
	}
	m.RUnlock()
}
func TestUnaryOp_Params(t *testing.T) {
	o := NewUnaryOp(context.Background())
	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		return nil
	})
	in := make(chan interface{})

	o.SetOperation(op)
	if o.op == nil {
		t.Fatal("process Elem not set")
	}

	o.SetConcurrency(4)
	if o.concurrency != 4 {
		t.Fatal("Concurrency not being set")
	}

	o.SetInput(in)
	if o.input == nil {
		t.Fatal("Input not being set")
	}

	if o.GetOutput == nil {
		t.Fatal("Output not set")
	}
}
func BenchmarkUnaryOp_Exec(b *testing.B) {
	ctx := context.Background()
	o := NewUnaryOp(ctx)
	N := b.N

	chanSize := func() int {
		if N == 1 {
			return N
		}
		return int(float64(0.5) * float64(N))
	}()

	in := make(chan interface{}, chanSize)
	o.SetInput(in)
	go func() {
		for i := 0; i < N; i++ {
			in <- testutil.GenWord()
		}
		close(in)
	}()

	counter := 0
	var m sync.RWMutex

	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		m.Lock()
		counter++
		m.Unlock()
		return data
	})
	o.SetOperation(op)

	// process output
	done := make(chan struct{})
	go func() {
		defer close(done)
		for _ = range o.GetOutput() {
		}
	}()

	if err := o.Exec(); err != nil {
		b.Fatal("Error during execution:", err)
	}

	select {
	case <-done:
	case <-time.After(time.Second * 60):
		b.Fatal("Took too long")
	}
	m.RLock()
	b.Logf("Input %d, counted %d", N, counter)
	if counter != N {
		b.Fatalf("Expected %d items processed,  got %d", N, counter)
	}
	m.RUnlock()
}
Exemple #4
0
func TestStream_InitGraph(t *testing.T) {
	src := newStrSrc([]string{"Hello", "World"})
	snk := newStrSink()
	op1 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		return nil
	})
	op2 := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		return nil
	})

	strm := New().From(src).To(snk)

	if err := strm.initGraph(); err != nil {
		t.Fatal(err)
	}

	if src.GetOutput() != snk.input {
		t.Fatal("Source not link to sink when no ops are present")
	}

	strm = New().From(src).Transform(op1).Transform(op2).To(snk)
	if err := strm.initGraph(); err != nil {
		t.Fatal(err)
	}

	if len(strm.ops) != 2 {
		t.Fatal("Not adding operations to stream")
	}

	if src.GetOutput() == snk.input {
		t.Fatal("Graph invalid, source skipping ops, linked to sink!")
	}

	if strm.ops[1].GetOutput() != snk.input {
		t.Fatal("Sink not linked to last element in graph")
	}

}
// Process is used to express general processing operation of incoming stream
// elements.  Function must be of the form "func(input)output", anything else
// will cause painic.
func (s *Stream) Process(f interface{}) *Stream {
	fntype := reflect.TypeOf(f)
	if err := s.isUnaryFuncForm(fntype); err != nil {
		panic(fmt.Sprintf("Op Process() failed: %s", err))
	}

	fnval := reflect.ValueOf(f)

	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		arg0 := reflect.ValueOf(data)
		result := fnval.Call([]reflect.Value{arg0})[0]
		return result.Interface()
	})

	return s.Transform(op)
}
func TestUnaryOp_Exec(t *testing.T) {
	ctx, _ := context.WithCancel(context.Background())
	o := NewUnaryOp(ctx)

	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		values := data.([]string)
		t.Logf("Processing data %v, sending %d", values, len(values))
		return len(values)
	})
	o.SetOperation(op)

	in := make(chan interface{})
	go func() {
		in <- []string{"A", "B", "C"}
		in <- []string{"D", "E"}
		in <- []string{"G"}
		close(in)
	}()
	o.SetInput(in)

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		for data := range o.GetOutput() {
			val, ok := data.(int)
			t.Logf("Got value %v", val)
			if !ok {
				t.Fatalf("Expecting type int, got %T, value %v", val, val)
			}
			if val != 3 && val != 2 && val != 1 {
				t.Fatalf("Expecting values 3, 2, or 1, but got %d", val)
			}
		}
	}()

	if err := o.Exec(); err != nil {
		t.Fatal(err)
	}

	select {
	case <-wait:
	case <-time.After(50 * time.Millisecond):
		t.Fatal("Took too long...")
	}
}
// FlatMap similar to Map, however, expected to return a slice of values
// to downstream operators.  The operator will flatten the slice and emit
// individually onto the stream. The FlatMap function must have the form
// "func(intput)[]output", anything else will be rejected
func (s *Stream) FlatMap(f interface{}) *Stream {
	fntype := reflect.TypeOf(f)
	if err := s.isUnaryFuncForm(fntype); err != nil {
		panic(fmt.Sprintf("Op FlatMap() failed: %s", err))
	}
	if fntype.Out(0).Kind() != reflect.Slice {
		panic(fmt.Sprintf("Op FlatMap() expects to return a slice of values"))
	}

	fnval := reflect.ValueOf(f)

	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		arg0 := reflect.ValueOf(data)
		result := fnval.Call([]reflect.Value{arg0})[0]
		return result.Interface()
	})

	s.Transform(op) // add flatmap as unary op
	s.ReStream()    // add streamop to unpack flatmap result
	return s
}
Exemple #8
0
func TestStream_BuilderMethods(t *testing.T) {
	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		return nil
	})

	st := New()
	st.
		From(newStrSrc([]string{"Hello", "World", "!!"})).
		To(newStrSink()).
		Transform(op)

	if st.source == nil {
		t.Fatal("From() not setting source")
	}
	if st.sink == nil {
		t.Fatal("To() not setting sink")
	}
	if len(st.ops) != 1 {
		t.Fatal("Operation not added to ops slice")
	}
}
// Filter takes a predicate func that filters the stream.
// Function must be of form "func(input)bool", anything else causes panic.
// If func returns true, current item continues downstream.
func (s *Stream) Filter(f interface{}) *Stream {
	fntype := reflect.TypeOf(f)
	if err := s.isUnaryFuncForm(fntype); err != nil {
		panic(fmt.Sprintf("Op Filter() failed :%s", err))
	}
	// ensure bool ret type
	if fntype.Out(0).Kind() != reflect.Bool {
		panic("Op Filter() must return a bool")
	}

	fnval := reflect.ValueOf(f)

	op := api.UnFunc(func(ctx context.Context, data interface{}) interface{} {
		arg0 := reflect.ValueOf(data)
		result := fnval.Call([]reflect.Value{arg0})[0]
		predicate := result.Bool()
		if !predicate {
			return nil
		}
		return data
	})
	return s.Transform(op)
}