Ejemplo n.º 1
0
func TestMakeInfoHandler(t *testing.T) {
	t.Parallel()

	c, _ := metafora.NewConsumer(&tc{stop: make(chan bool)}, nil, metafora.DumbBalancer)
	defer c.Shutdown()
	now := time.Now().Truncate(time.Second)

	resp := httptest.NewRecorder()
	MakeInfoHandler(c, now)(resp, nil)

	info := InfoResponse{}
	if err := json.Unmarshal(resp.Body.Bytes(), &info); err != nil {
		t.Fatalf("Error unmarshalling response body: %v", err)
	}
	if info.Frozen {
		t.Errorf("Consumer should not start frozen.")
	}
	if !info.Started.Equal(now) {
		t.Errorf("Started time %s != %s", info.Started, now)
	}
	if info.Name != "tc" {
		t.Errorf("Node name %q != tc", info.Name)
	}
	if len(info.Tasks) != 0 {
		t.Errorf("Unexpected tasks: %v", info.Tasks)
	}
}
Ejemplo n.º 2
0
func TestEmbeddedShutdown(t *testing.T) {
	const n = 4
	runs := make(chan int, n)
	stops := make(chan int, n)
	thfunc := metafora.SimpleHandler(func(_ metafora.Task, s <-chan bool) bool {
		runs <- 1
		select {
		case <-s:
			stops <- 1
			return false
		case <-time.After(time.Second * 3):
			return true
		}
	})

	coord, client := NewEmbeddedPair("testnode")
	runner, _ := metafora.NewConsumer(coord, thfunc, metafora.DumbBalancer)

	go runner.Run()

	// len(tasks) must == n
	tasks := []string{"one", "two", "three", "four"}

	// submit tasks
	for _, taskid := range tasks {
		err := client.SubmitTask(metafora.NewTask(taskid))
		if err != nil {
			t.Fatalf("Expected no error, got %v", err)
		}
	}

	// make sure all 4 start
	for i := 0; i < n; i++ {
		<-runs
	}

	// tell them to stop
	runner.Shutdown()

	// make sure all 4 stop
	for i := 0; i < n; i++ {
		<-stops
	}
}
Ejemplo n.º 3
0
func TestEmbedded(t *testing.T) {

	tc := newTestCounter()
	adds := make(chan string, 4)

	thfunc := metafora.SimpleHandler(func(task metafora.Task, _ <-chan bool) bool {
		tc.Add(task.ID())
		adds <- task.ID()
		return true
	})

	coord, client := NewEmbeddedPair("testnode")
	runner, _ := metafora.NewConsumer(coord, thfunc, metafora.DumbBalancer)

	go runner.Run()

	for _, taskid := range []string{"one", "two", "three", "four"} {
		err := client.SubmitTask(metafora.NewTask(taskid))
		if err != nil {
			t.Fatalf("Expected no error, got %v", err)
		}
	}

	deadline := time.Now().Add(500 * time.Millisecond)
	for time.Now().Before(deadline) {
		if len(adds) == 4 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	if len(adds) != 4 {
		t.Errorf("Handlers didn't run in expected amount of time")
	}
	runner.Shutdown()

	runs := tc.Runs()
	if len(runs) != 4 {
		t.Fatalf("Expected 4 runs, got %d", len(runs))
	}

}
Ejemplo n.º 4
0
// TestExpiration ensures that expired claims get reclaimed properly.
func TestExpiration(t *testing.T) {
	t.Parallel()
	coord, conf := setupEtcd(t)
	claims := make(chan int, 10)
	hf := metafora.HandlerFunc(metafora.SimpleHandler(func(_ metafora.Task, stop <-chan bool) bool {
		claims <- 1
		<-stop
		return true
	}))
	consumer, err := metafora.NewConsumer(coord, hf, metafora.DumbBalancer)
	if err != nil {
		t.Fatalf("Error creating consumer: %+v", err)
	}
	client, _ := testutil.NewEtcdClient(t)

	_, err = client.Create(path.Join(conf.Namespace, TasksPath, "abc", OwnerMarker), `{"node":"--"}`, 1)
	if err != nil {
		t.Fatalf("Error creating fake claim: %v", err)
	}

	defer consumer.Shutdown()
	go consumer.Run()

	// Wait for claim to expire and coordinator to pick up task
	select {
	case <-claims:
		// Task claimed!
	case <-time.After(5 * time.Second):
		t.Fatal("Task not claimed long after it should have been.")
	}

	tasks := consumer.Tasks()
	if len(tasks) != 1 {
		t.Fatalf("Expected 1 task to be claimed but found: %v", tasks)
	}
}
Ejemplo n.º 5
0
func main() {
	mlvl := metafora.LogLevelInfo
	hostname, _ := os.Hostname()

	peers := flag.String("etcd", "http://127.0.0.1:2379", "comma delimited etcd peer list")
	namespace := flag.String("namespace", "koalemos", "metafora namespace")
	name := flag.String("name", hostname, "node name or empty for automatic")
	loglvl := flag.String("log", mlvl.String(), "set log level: [debug], info, warn, error")
	flag.Parse()

	hosts := strings.Split(*peers, ",")
	etcdc := etcd.NewClient(hosts)

	switch strings.ToLower(*loglvl) {
	case "debug":
		mlvl = metafora.LogLevelDebug
	case "info":
		mlvl = metafora.LogLevelInfo
	case "warn":
		mlvl = metafora.LogLevelWarn
	case "error":
		mlvl = metafora.LogLevelError
	default:
		metafora.Warnf("Invalid log level %q - using %s", *loglvl, mlvl)
	}
	metafora.SetLogLevel(mlvl)

	conf := m_etcd.NewConfig(*name, *namespace, hosts)

	// Replace NewTask func with one that returns a *koalemos.Task
	conf.NewTaskFunc = func(id, value string) metafora.Task {
		t := koalemos.NewTask(id)
		if value == "" {
			return t
		}
		if err := json.Unmarshal([]byte(value), t); err != nil {
			metafora.Errorf("Unable to unmarshal task %s: %v", t.ID(), err)
			return nil
		}
		return t
	}

	hfunc := makeHandlerFunc(etcdc)
	ec, err := m_etcd.NewEtcdCoordinator(conf)
	if err != nil {
		metafora.Errorf("Error creating etcd coordinator: %v", err)
	}

	bal := m_etcd.NewFairBalancer(conf)
	c, err := metafora.NewConsumer(ec, hfunc, bal)
	if err != nil {
		metafora.Errorf("Error creating consumer: %v", err)
		os.Exit(2)
	}
	metafora.Infof(
		"Starting koalsmosd with etcd=%s; namespace=%s; name=%s; loglvl=%s",
		*peers, conf.Namespace, conf.Name, mlvl)
	consumerRunning := make(chan struct{})
	go func() {
		defer close(consumerRunning)
		c.Run()
	}()

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, os.Interrupt, os.Kill, syscall.SIGTERM)
	select {
	case s := <-sigC:
		metafora.Infof("Received signal %s, shutting down", s)
	case <-consumerRunning:
		metafora.Warn("Consumer exited. Shutting down.")
	}
	c.Shutdown()
	metafora.Info("Shutdown")
}
Ejemplo n.º 6
0
Archivo: grid.go Proyecto: mdmarek/grid
// Start the grid. Actors that were stopped from a previous exit
// of the grid but returned a "done" status of false will start
// to be scheduled. New actors can be scheduled with StartActor.
func (g *grid) Start() (<-chan bool, error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	// Start only once.
	if g.started {
		return g.exit, nil
	}

	// Use the hostname as the node identifier.
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	nodeid := fmt.Sprintf("%s-%s", hostname, g.name)

	// Define the metafora new task function and config.
	conf := m_etcd.NewConfig(nodeid, g.name, g.etcdservers)
	conf.NewTaskFunc = func(id, value string) metafora.Task {
		def := NewActorDef(id)
		err := json.Unmarshal([]byte(value), def)
		if err != nil {
			log.Printf("error: failed to schedule actor: %v, error: %v", id, err)
			return nil
		}
		a, err := g.maker.MakeActor(def)
		if err != nil {
			log.Printf("error: failed to schedule actor: %v, error: %v", id, err)
			return nil
		}

		return newHandler(g.fork(), a)
	}

	// Create the metafora etcd coordinator.
	ec, err := m_etcd.NewEtcdCoordinator(conf)
	if err != nil {
		return nil, err
	}

	// Create the metafora consumer.
	c, err := metafora.NewConsumer(ec, handler(etcd.NewClient(g.etcdservers)), m_etcd.NewFairBalancer(conf))
	if err != nil {
		return nil, err
	}
	g.metaconsumer = c
	g.metaclient = m_etcd.NewClient(g.name, g.etcdservers)
	g.stopped = false
	g.started = true

	// Close the exit channel when metafora thinks
	// an exit is needed.
	go func() {
		defer close(g.exit)
		g.metaconsumer.Run()
	}()

	for i := 0; i < 2*runtime.NumCPU(); i++ {
		natsconn, err := g.newNatsConn()
		if err != nil {
			return nil, err
		}
		g.natsconnpool[i] = natsconn
		if i == 0 {
			g.natsconn = g.natsconnpool[0]
		}
	}

	return g.exit, nil
}
Ejemplo n.º 7
0
// TestSleepTest is an integration test for all of m_etcd's components.
//
func TestSleepTest(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()
	const namespace = "sleeptest-metafora"
	const sleepingtasks = "sleeping-task1"

	etcdc.Delete(namespace, recursive)

	holdtask := make(chan bool)
	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {

		if task.ID() == sleepingtasks {
			sleeptil := 5 * time.Second
			nextstarttime := (time.Now().Add(sleeptil))
			t.Logf("sleeping task:%v sleepfor:%v", task, nextstarttime)
			<-holdtask
			return statemachine.SleepMessage(nextstarttime)
		}

		cmd := <-cmds
		t.Logf("non sleeping task:%v", task)

		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(name, ns, hosts)
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go func() {
			cons.Run()
			t.Logf("Consumer:%s exited.", name)
		}()
		return cons
	}

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	// Start 2 consumers
	cons1 := newC("node1", namespace)
	cons2 := newC("node2", namespace)

	// Create clients and start some tests
	cliA := m_etcd.NewClient(namespace, hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(sleepingtasks, "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2)

	holdtask <- true
	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2) // not sure if this should be true or false.

	wait1 := make(chan bool)
	go func() {
		defer close(wait1)
		// Shutdown
		cons1.Shutdown()
		cons2.Shutdown()
	}()

	timeout := time.NewTimer(1 * time.Second)
	select {
	case <-wait1:
	case <-timeout.C:
		t.Fatalf("failed waiting for shutdown")
	}

	//	make sure all tasks are released
	for _, c := range []*metafora.Consumer{cons1, cons2} {
		tasks := c.Tasks()
		for _, work := range tasks {
			t.Fatalf("work id %v is still running", work)
		}
	}
}
Ejemplo n.º 8
0
// TestAll is an integration test for all of m_etcd's components.
//
// While huge integration tests like this are rarely desirable as they can be
// overly fragile and complex, I found myself manually repeating the tests I've
// automated here over and over. This is far more reliable than expecting
// developers to do adhoc testing of all of the m_etcd package's features.
func TestAll(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()

	etcdc.Delete("test-a", recursive)
	etcdc.Delete("test-b", recursive)

	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {
		cmd := <-cmds
		if task.ID() == "error-test" {
			return statemachine.ErrorMessage(errors.New("error-test"))
		}
		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(name, ns, hosts)
		conf.Name = name
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go cons.Run()
		return cons
	}
	// Start 4 consumers, 2 per namespace
	cons1a := newC("node1", "test-a")
	cons2a := newC("node2", "test-a")
	cons1b := newC("node1", "test-b")
	cons2b := newC("node2", "test-b")

	// Create clients and start some tests
	cliA := m_etcd.NewClient("test-a", hosts)
	cliB := m_etcd.NewClient("test-b", hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}
	if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to b: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(250 * time.Millisecond)

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	assertRunning("task1", cons1a, cons2a)
	assertRunning("task1", cons1b, cons2b)

	// Kill task1 in A
	{
		cmdr := m_etcd.NewCommander("test-a", etcdc)
		if err := cmdr.Send("task1", statemachine.KillMessage()); err != nil {
			t.Fatalf("Error sending kill to task1: %v", err)
		}
		time.Sleep(250 * time.Millisecond)

		for _, c := range []*metafora.Consumer{cons1a, cons2a} {
			tasks := c.Tasks()
			if len(tasks) != 0 {
				t.Fatalf("Expected no tasks but found: %d", len(tasks))
			}
		}
	}

	// Submit a bunch of tasks to A
	{
		tasks := []string{"task2", "task3", "task4", "task5", "task6", "task7"}
		for _, tid := range tasks {
			if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to A: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(800 * time.Millisecond)

		// Ensure they're balanced
		if err := cliA.SubmitCommand("node1", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}
		time.Sleep(800 * time.Millisecond)
		if err := cliA.SubmitCommand("node2", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}

		a1tasks := cons1a.Tasks()
		a2tasks := cons2a.Tasks()
		for _, task := range a1tasks {
			metafora.Debug("A1: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		for _, task := range a2tasks {
			metafora.Debug("A2: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		time.Sleep(800 * time.Millisecond)

		a1tasks = cons1a.Tasks()
		a2tasks = cons2a.Tasks()
		if len(a1tasks) < 2 || len(a1tasks) > 4 || len(a2tasks) < 2 || len(a2tasks) > 4 {
			t.Fatalf("Namespace A isn't fairly balanced: node1: %d; node2: %d", len(a1tasks), len(a2tasks))
		}

		// Shutting down a consumer should migrate all tasks to the other
		cons1a.Shutdown()
		time.Sleep(800 * time.Millisecond)

		a2tasks = cons2a.Tasks()
		if len(a2tasks) != len(tasks) {
			t.Fatalf("Consumer 2a should have received all %d tasks but only has %d.", len(tasks), len(a2tasks))
		}
	}

	// Use Namespace B to check Error state handling
	{
		tasks := []string{"task8", "error-test"}
		for _, tid := range tasks {
			if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to B: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(time.Second)

		n := len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 3 {
			t.Fatalf("Expected B to be running 3 tasks but found %d", n)
		}

		// Resuming error-test 8*2 times should cause it to be failed
		cmdr := m_etcd.NewCommander("test-b", etcdc)
		for i := 0; i < statemachine.DefaultErrMax*2; i++ {
			if err := cmdr.Send("error-test", statemachine.RunMessage()); err != nil {
				t.Fatalf("Unexpected error resuming error-test in B: %v", err)
			}
			time.Sleep(500 * time.Millisecond)
		}

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}

		// Resubmitting a failed task shouldn't error but also shouldn't run.
		if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("error-test", "")); err != nil {
			t.Fatalf("Error resubmitting error-test task to B: %v", err)
		}

		// Give the statemachine a moment to load the initial state and exit
		time.Sleep(time.Second)

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}
	}

	// Shutdown
	cons2a.Shutdown()
	cons1b.Shutdown()
	cons2b.Shutdown()

	// Make sure everything is cleaned up
	respA, err := etcdc.Get("/test-a/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}
	respB, err := etcdc.Get("/test-b/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}

	nodes := []*etcd.Node{}
	nodes = append(nodes, respA.Node.Nodes...)
	nodes = append(nodes, respB.Node.Nodes...)
	for _, node := range nodes {
		if len(node.Nodes) > 1 {
			t.Fatalf("%s has %d (>1) nodes. First 2: %s, %s", node.Key, len(node.Nodes), node.Nodes[0].Key, node.Nodes[1].Key)
		}
	}
}
Ejemplo n.º 9
0
// Fair balancer shouldn't consider a shutting-down node
// See https://github.com/lytics/metafora/issues/92
func TestFairBalancerShutdown(t *testing.T) {
	coord1, conf1 := setupEtcd(t)
	conf2 := conf1.Copy()
	conf2.Name = "node2"
	coord2, _ := NewEtcdCoordinator(conf2)

	cli := NewClient(conf1.Namespace, conf1.Hosts)

	// This handler always returns immediately
	h1 := metafora.SimpleHandler(func(task metafora.Task, stop <-chan bool) bool {
		metafora.Debugf("H1 Starting %s", task.ID())
		<-stop
		metafora.Debugf("H1 Stopping %s", task.ID())
		return false // never done
	})

	// Block forever on a single task
	stop2 := make(chan struct{})
	stopr := make(chan chan struct{}, 1)
	stopr <- stop2
	h2 := metafora.SimpleHandler(func(task metafora.Task, stop <-chan bool) bool {
		metafora.Debugf("H2 Starting %s", task.ID())
		blockchan, ok := <-stopr
		if ok {
			<-blockchan
		}
		<-stop
		metafora.Debugf("H2 Stopping %s", task.ID())
		return false // never done
	})

	// Create two consumers
	b1 := NewFairBalancer(conf1)
	con1, err := metafora.NewConsumer(coord1, h1, b1)
	if err != nil {
		t.Fatal(err)
	}

	b2 := NewFairBalancer(conf2)
	con2, err := metafora.NewConsumer(coord2, h2, b2)
	if err != nil {
		t.Fatal(err)
	}

	// Start the first and let it claim a bunch of tasks
	go con1.Run()
	defer con1.Shutdown()
	cli.SubmitTask(DefaultTaskFunc("t1", ""))
	cli.SubmitTask(DefaultTaskFunc("t2", ""))
	cli.SubmitTask(DefaultTaskFunc("t3", ""))
	cli.SubmitTask(DefaultTaskFunc("t4", ""))
	cli.SubmitTask(DefaultTaskFunc("t5", ""))
	cli.SubmitTask(DefaultTaskFunc("t6", ""))

	time.Sleep(500 * time.Millisecond)

	if len(con1.Tasks()) != 6 {
		t.Fatalf("con1 should have claimed 6 tasks: %d", len(con1.Tasks()))
	}

	// Start the second consumer and force the 1st to rebalance
	go con2.Run()

	close(stopr)

	// Wait for node to startup and register
	time.Sleep(500 * time.Millisecond)

	cli.SubmitCommand(conf1.Name, metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	c1Tasks := con1.Tasks()
	c2Tasks := con2.Tasks()
	if len(c1Tasks) != 4 || len(c2Tasks) != 2 {
		t.Fatalf("expected consumers to have 4|2 tasks: %d|%d", len(c1Tasks), len(c2Tasks))
	}

	// Make sure that balancing the other node does nothing
	cli.SubmitCommand("node2", metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	c1Tasks2 := con1.Tasks()
	c2Tasks2 := con2.Tasks()
	if len(c1Tasks2) != 4 || len(c2Tasks2) != 2 {
		t.Fatalf("expected consumers to have 4|2 tasks: %d|%d", len(c1Tasks2), len(c2Tasks2))
	}
	for i := 0; i < 4; i++ {
		if c1Tasks[i] != c1Tasks2[i] {
			t.Errorf("task mismatch: %s != %s", c1Tasks[i], c1Tasks2[i])
		}
	}
	for i := 0; i < 2; i++ {
		if c2Tasks[i] != c2Tasks2[i] {
			t.Errorf("task mismatch: %s != %s", c2Tasks[i], c2Tasks2[i])
		}
	}

	// Second consumer should block on a single task forever
	// Rebalancing the first node should then cause it to pickup all but
	// one task
	c2stop := make(chan struct{})
	go func() {
		con2.Shutdown()
		close(c2stop)
	}()

	time.Sleep(500 * time.Millisecond)

	cli.SubmitCommand(conf1.Name, metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	c1Tasks3 := con1.Tasks()
	c2Tasks3 := con2.Tasks()
	if len(c1Tasks3) != 5 || len(c2Tasks3) != 1 {
		t.Fatalf("Expected consumers to have 5|1 tasks: %d|%d", len(c1Tasks3), len(c2Tasks3))
	}

	// Now stop blocking task, rebalance and make sure the first node picked up the remaining
	close(stop2)

	time.Sleep(500 * time.Millisecond)
	// Consumer 2 should stop now
	<-c2stop

	cli.SubmitCommand(conf1.Name, metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	// con2 is out of the picture. con1 has all the tasks.
	c1Tasks4 := con1.Tasks()
	c2Tasks4 := con2.Tasks()
	if len(c1Tasks4) != 6 || len(c2Tasks4) != 0 {
		t.Fatalf("Expected consumers to have 6|0 tasks: %d|%d", len(c1Tasks4), len(c2Tasks4))
	}
}
Ejemplo n.º 10
0
func TestFairBalancer(t *testing.T) {
	t.Parallel()
	coord1, conf1 := setupEtcd(t)
	conf2 := conf1.Copy()
	conf2.Name = "coord2"
	coord2, _ := NewEtcdCoordinator(conf2)

	cli := NewClient(conf1.Namespace, conf1.Hosts)

	h := metafora.SimpleHandler(func(task metafora.Task, stop <-chan bool) bool {
		metafora.Debugf("Starting %s", task.ID())
		<-stop
		metafora.Debugf("Stopping %s", task.ID())
		return false // never done
	})

	// Create two consumers
	b1 := NewFairBalancer(conf1)
	con1, err := metafora.NewConsumer(coord1, h, b1)
	if err != nil {
		t.Fatal(err)
	}

	b2 := NewFairBalancer(conf2)
	con2, err := metafora.NewConsumer(coord2, h, b2)
	if err != nil {
		t.Fatal(err)
	}

	// Start the first and let it claim a bunch of tasks
	go con1.Run()
	defer con1.Shutdown()
	cli.SubmitTask(DefaultTaskFunc("t1", ""))
	cli.SubmitTask(DefaultTaskFunc("t2", ""))
	cli.SubmitTask(DefaultTaskFunc("t3", ""))
	cli.SubmitTask(DefaultTaskFunc("t4", ""))
	cli.SubmitTask(DefaultTaskFunc("t5", ""))
	cli.SubmitTask(DefaultTaskFunc("t6", ""))

	time.Sleep(1 * time.Second)

	if len(con1.Tasks()) != 6 {
		t.Fatalf("con1 should have claimed 6 tasks: %d", len(con1.Tasks()))
	}

	// Start the second consumer and force the 1st to rebalance
	go con2.Run()
	defer con2.Shutdown()

	// Wait for node to startup and register
	time.Sleep(1 * time.Second)

	cli.SubmitCommand(conf1.Name, metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	c1Tasks := con1.Tasks()
	c2Tasks := con2.Tasks()
	if len(c1Tasks) != 4 || len(c2Tasks) != 2 {
		t.Fatalf("expected consumers to have 4|2 tasks: %d|%d", len(c1Tasks), len(c2Tasks))
	}

	// Finally make sure that balancing the other node does nothing
	cli.SubmitCommand("node2", metafora.CommandBalance())

	time.Sleep(2 * time.Second)

	c1Tasks2 := con1.Tasks()
	c2Tasks2 := con2.Tasks()
	if len(c1Tasks2) != 4 || len(c2Tasks2) != 2 {
		t.Fatalf("expected consumers to have 4|2 tasks: %d|%d", len(c1Tasks2), len(c2Tasks2))
	}
	for i := 0; i < 4; i++ {
		if c1Tasks[i] != c1Tasks2[i] {
			t.Errorf("task mismatch: %s != %s", c1Tasks[i], c1Tasks2[i])
		}
	}
	for i := 0; i < 2; i++ {
		if c2Tasks[i] != c2Tasks2[i] {
			t.Errorf("task mismatch: %s != %s", c2Tasks[i], c2Tasks2[i])
		}
	}
}
Ejemplo n.º 11
0
// TestNodeRefresher ensures the node refresher properly updates the TTL on the
// node directory in etcd and shuts down the entire consumer on error.
func TestNodeRefresher(t *testing.T) {
	t.Parallel()
	_, conf := setupEtcd(t)

	// Use a custom node path ttl
	conf.NodeTTL = 3
	coord, err := NewEtcdCoordinator(conf)
	if err != nil {
		t.Fatalf("Error creating coordinator: %v", err)
	}

	hf := metafora.HandlerFunc(nil) // we won't be handling any tasks
	consumer, err := metafora.NewConsumer(coord, hf, metafora.DumbBalancer)
	if err != nil {
		t.Fatalf("Error creating consumer: %+v", err)
	}
	client, _ := testutil.NewEtcdClient(t)

	defer consumer.Shutdown()
	runDone := make(chan struct{})
	go func() {
		consumer.Run()
		close(runDone)
	}()

	nodePath := path.Join(conf.Namespace, NodesPath, conf.Name)
	ttl := int64(-1)
	deadline := time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		resp, _ := client.Get(nodePath, false, false)
		if resp != nil && resp.Node.Dir {
			ttl = resp.Node.TTL
			break
		}
		time.Sleep(250 * time.Millisecond)
	}
	if ttl == -1 {
		t.Fatalf("Node path %s not found.", nodePath)
	}
	if ttl < 1 || ttl > 3 {
		t.Fatalf("Expected TTL to be between 1 and 3, found: %d", ttl)
	}

	// Let it refresh once to make sure that works
	time.Sleep(time.Duration(ttl) * time.Second)
	ttl = -1
	deadline = time.Now().Add(3 * time.Second)
	for time.Now().Before(deadline) {
		resp, _ := client.Get(nodePath, false, false)
		if resp != nil && resp.Node.Dir {
			ttl = resp.Node.TTL
			break
		}
		time.Sleep(250 * time.Millisecond)
	}
	if ttl == -1 {
		t.Fatalf("Node path %s not found.", nodePath)
	}

	// Now remove the node out from underneath the refresher to cause it to fail
	if _, err := client.Delete(nodePath, true); err != nil {
		t.Fatalf("Unexpected error deleting %s: %+v", nodePath, err)
	}

	select {
	case <-runDone:
		// success! run exited
	case <-time.After(5 * time.Second):
		t.Fatal("Consumer didn't exit even though node directory disappeared!")
	}
}