// TestRequestDataEpochMismatch creates a scenario where data request happened
// with two different epochs. In this case, the server should back pressure and
// request client should get notified and return error.
func TestRequestDataEpochMismatch(t *testing.T) {
t.Skip("TODO")
job := "TestRequestDataEpochMismatch"
etcdURLs := []string{"http://localhost:4001"}
ctl := controller.New(job, etcd.NewClient(etcdURLs), 1, []string{"Parents", "Children"})
ctl.InitEtcdLayout()
defer ctl.DestroyEtcdLayout()
fw := &framework{
name: job,
etcdURLs: etcdURLs,
ln: createListener(t),
}
var wg sync.WaitGroup
fw.SetTaskBuilder(&testableTaskBuilder{
setupLatch: &wg,
})
fw.AddLinkage("Parents", topo.NewTreeTopologyOfParent(1, 1))
fw.AddLinkage("Children", topo.NewTreeTopologyOfChildren(1, 1))
wg.Add(1)
go fw.Start()
wg.Wait()
defer fw.ShutdownJob()
addr, err := etcdutil.GetAddress(fw.etcdClient, job, fw.GetTaskID())
if err != nil {
t.Fatalf("GetAddress failed: %v", err)
}
addr = addr
// _, err = frameworkhttp.RequestData(addr, "Parents", "req", 0, fw.GetTaskID(), 10, fw.GetLogger())
// if err != frameworkhttp.ErrReqEpochMismatch {
// t.Fatalf("error want = %v, but get = (%)", frameworkhttp.ErrReqEpochMismatch, err.Error())
// }
}
func (f *framework) sendRequest(dr *dataRequest) {
addr, err := etcdutil.GetAddress(f.etcdClient, f.name, dr.taskID)
if err != nil {
f.log.Printf("getAddress(%d) failed: %v", dr.taskID, err)
go f.retrySendRequest(dr)
return
}
// TODO: reuse ClientConn and reply message.
// The grpc.WithTimeout would help detect any disconnection in failfast.
// Otherwise grpc.Invoke will keep retrying.
cc, err := grpc.Dial(addr, grpc.WithTimeout(heartbeatInterval))
// we need to retry if some task failed and there is a temporary Get request failure.
if err != nil {
f.log.Printf("grpc.Dial to task %d (addr: %s) failed: %v", dr.taskID, addr, err)
// Should retry for other errors.
go f.retrySendRequest(dr)
return
}
defer cc.Close()
if dr.retry {
f.log.Printf("retry data request %s to task %d, addr %s", dr.method, dr.taskID, addr)
} else {
f.log.Printf("data request %s to task %d, addr %s", dr.method, dr.taskID, addr)
}
reply := f.task.CreateOutputMessage(dr.method)
err = grpc.Invoke(dr.ctx, dr.method, dr.input, reply, cc)
if err != nil {
f.log.Printf("grpc.Invoke to task %d (addr: %s), method: %s, failed: %v", dr.taskID, addr, dr.method, err)
go f.retrySendRequest(dr)
return
}
select {
case f.dataRespChan <- &dataResponse{
epoch: dr.epoch,
taskID: dr.taskID,
method: dr.method,
input: dr.input,
output: reply,
}:
case <-dr.ctx.Done():
f.log.Printf("abort data response, to %d, epoch %d, method %s", dr.taskID, dr.epoch, dr.method)
}
}