Example #1
0
// TestRequestDataEpochMismatch creates a scenario where data request happened
// with two different epochs. In this case, the server should back pressure and
// request client should get notified and return error.
func TestRequestDataEpochMismatch(t *testing.T) {
	t.Skip("TODO")
	job := "TestRequestDataEpochMismatch"
	etcdURLs := []string{"http://localhost:4001"}
	ctl := controller.New(job, etcd.NewClient(etcdURLs), 1, []string{"Parents", "Children"})
	ctl.InitEtcdLayout()
	defer ctl.DestroyEtcdLayout()

	fw := &framework{
		name:     job,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}
	var wg sync.WaitGroup
	fw.SetTaskBuilder(&testableTaskBuilder{
		setupLatch: &wg,
	})
	fw.AddLinkage("Parents", topo.NewTreeTopologyOfParent(1, 1))
	fw.AddLinkage("Children", topo.NewTreeTopologyOfChildren(1, 1))
	wg.Add(1)
	go fw.Start()
	wg.Wait()
	defer fw.ShutdownJob()

	addr, err := etcdutil.GetAddress(fw.etcdClient, job, fw.GetTaskID())
	if err != nil {
		t.Fatalf("GetAddress failed: %v", err)
	}
	addr = addr
	// _, err = frameworkhttp.RequestData(addr, "Parents", "req", 0, fw.GetTaskID(), 10, fw.GetLogger())
	// if err != frameworkhttp.ErrReqEpochMismatch {
	// 	t.Fatalf("error want = %v, but get = (%)", frameworkhttp.ErrReqEpochMismatch, err.Error())
	// }
}
Example #2
0
func (f *framework) sendRequest(dr *dataRequest) {
	addr, err := etcdutil.GetAddress(f.etcdClient, f.name, dr.taskID)
	if err != nil {
		f.log.Printf("getAddress(%d) failed: %v", dr.taskID, err)
		go f.retrySendRequest(dr)
		return
	}
	// TODO: reuse ClientConn and reply message.
	// The grpc.WithTimeout would help detect any disconnection in failfast.
	// Otherwise grpc.Invoke will keep retrying.
	cc, err := grpc.Dial(addr, grpc.WithTimeout(heartbeatInterval))
	// we need to retry if some task failed and there is a temporary Get request failure.
	if err != nil {
		f.log.Printf("grpc.Dial to task %d (addr: %s) failed: %v", dr.taskID, addr, err)
		// Should retry for other errors.
		go f.retrySendRequest(dr)
		return
	}
	defer cc.Close()
	if dr.retry {
		f.log.Printf("retry data request %s to task %d, addr %s", dr.method, dr.taskID, addr)
	} else {
		f.log.Printf("data request %s to task %d, addr %s", dr.method, dr.taskID, addr)
	}
	reply := f.task.CreateOutputMessage(dr.method)
	err = grpc.Invoke(dr.ctx, dr.method, dr.input, reply, cc)
	if err != nil {
		f.log.Printf("grpc.Invoke to task %d (addr: %s), method: %s, failed: %v", dr.taskID, addr, dr.method, err)
		go f.retrySendRequest(dr)
		return
	}

	select {
	case f.dataRespChan <- &dataResponse{
		epoch:  dr.epoch,
		taskID: dr.taskID,
		method: dr.method,
		input:  dr.input,
		output: reply,
	}:
	case <-dr.ctx.Done():
		f.log.Printf("abort data response, to %d, epoch %d, method %s", dr.taskID, dr.epoch, dr.method)
	}
}