func main() {
programType := flag.String("type", "", "(c) controller or (t) task")
job := flag.String("job", "", "job name")
etcdURLs := []string{"http://localhost:4001"}
flag.Parse()
if *job == "" {
log.Fatalf("Please specify a job name")
}
ntask := uint64(2)
switch *programType {
case "c":
log.Printf("controller")
controller := controller.New(*job, etcd.NewClient(etcdURLs), ntask)
controller.Start()
controller.WaitForJobDone()
case "t":
log.Printf("task")
bootstrap := framework.NewBootStrap(*job, etcdURLs, createListener(), nil)
taskBuilder := ®ression.SimpleTaskBuilder{
GDataChan: make(chan int32, 11),
NumberOfIterations: 10,
MasterConfig: map[string]string{"writefile": "result.txt"},
}
bootstrap.SetTaskBuilder(taskBuilder)
bootstrap.AddLinkage("Parents" : topo.NewTreeTopologyOfParent(2, ntask))
bootstrap.AddLinkage("Children" : topo.NewTreeTopologyOfChildren(2, ntask))
bootstrap.Start()
default:
log.Fatal("Please choose a type: (c) controller, (t) task")
}
}
// TestRequestDataEpochMismatch creates a scenario where data request happened
// with two different epochs. In this case, the server should back pressure and
// request client should get notified and return error.
func TestRequestDataEpochMismatch(t *testing.T) {
t.Skip("TODO")
job := "TestRequestDataEpochMismatch"
etcdURLs := []string{"http://localhost:4001"}
ctl := controller.New(job, etcd.NewClient(etcdURLs), 1, []string{"Parents", "Children"})
ctl.InitEtcdLayout()
defer ctl.DestroyEtcdLayout()
fw := &framework{
name: job,
etcdURLs: etcdURLs,
ln: createListener(t),
}
var wg sync.WaitGroup
fw.SetTaskBuilder(&testableTaskBuilder{
setupLatch: &wg,
})
fw.AddLinkage("Parents", topo.NewTreeTopologyOfParent(1, 1))
fw.AddLinkage("Children", topo.NewTreeTopologyOfChildren(1, 1))
wg.Add(1)
go fw.Start()
wg.Wait()
defer fw.ShutdownJob()
addr, err := etcdutil.GetAddress(fw.etcdClient, job, fw.GetTaskID())
if err != nil {
t.Fatalf("GetAddress failed: %v", err)
}
addr = addr
// _, err = frameworkhttp.RequestData(addr, "Parents", "req", 0, fw.GetTaskID(), 10, fw.GetLogger())
// if err != frameworkhttp.ErrReqEpochMismatch {
// t.Fatalf("error want = %v, but get = (%)", frameworkhttp.ErrReqEpochMismatch, err.Error())
// }
}
// This is used to show how to drive the network.
func driveWithTreeTopo(t *testing.T, jobName string, etcds []string, ntask uint64, taskBuilder taskgraph.TaskBuilder) {
drive(
t,
jobName,
etcds,
taskBuilder,
map[string]taskgraph.Topology{
"Parents": topo.NewTreeTopologyOfParent(2, ntask),
"Children": topo.NewTreeTopologyOfChildren(2, ntask),
},
)
}
// TestFrameworkFlagMetaReady and TestFrameworkDataRequest test basic workflows of
// framework impl. It uses a scenario with two nodes: 0 as parent, 1 as child.
// The basic idea is that when parent tries to talk to child and vice versa,
// there will be some data transferring and captured by application task.
// Here we have implemented a helper user task to capture those data, test if
// it's passed from framework correctly and unmodified.
func TestFrameworkFlagMeta(t *testing.T) {
appName := "TestFrameworkFlagMeta"
etcdURLs := []string{"http://localhost:4001"}
// launch controller to setup etcd layout
ctl := controller.New(appName, etcd.NewClient(etcdURLs), 2, []string{"Parents", "Children"})
if err := ctl.InitEtcdLayout(); err != nil {
t.Fatalf("initEtcdLayout failed: %v", err)
}
defer ctl.DestroyEtcdLayout()
pDataChan := make(chan *tDataBundle, 1)
cDataChan := make(chan *tDataBundle, 1)
// simulate two tasks on two nodes -- 0 and 1
// 0 is parent, 1 is child
f0 := &framework{
name: appName,
etcdURLs: etcdURLs,
ln: createListener(t),
}
f1 := &framework{
name: appName,
etcdURLs: etcdURLs,
ln: createListener(t),
}
var wg sync.WaitGroup
taskBuilder := &testableTaskBuilder{
cDataChan: cDataChan,
pDataChan: pDataChan,
setupLatch: &wg,
}
f0.SetTaskBuilder(taskBuilder)
f0.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
f0.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
f1.SetTaskBuilder(taskBuilder)
f1.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
f1.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
taskBuilder.setupLatch.Add(2)
go f0.Start()
go f1.Start()
taskBuilder.setupLatch.Wait()
if f0.GetTaskID() != 0 {
f0, f1 = f1, f0
}
defer f0.ShutdownJob()
tests := []struct {
cMeta string
pMeta string
}{
{"parent", "child"},
{"ParamReady", "GradientReady"},
}
ctx := context.WithValue(context.Background(), epochKey, uint64(0))
for i, tt := range tests {
// 0: F#FlagChildMetaReady -> 1: T#ParentMetaReady
f0.FlagMeta(ctx, "Children", tt.cMeta)
// from child(1)'s view
data := <-cDataChan
expected := &tDataBundle{id: 0, meta: tt.cMeta}
if !reflect.DeepEqual(data, expected) {
t.Errorf("#%d: data bundle want = %v, get = %v", i, expected, data)
}
// 1: F#FlagParentMetaReady -> 0: T#ChildMetaReady
f1.FlagMeta(ctx, "Parents", tt.pMeta)
// from parent(0)'s view
data = <-pDataChan
expected = &tDataBundle{id: 1, meta: tt.pMeta}
if !reflect.DeepEqual(data, expected) {
t.Errorf("#%d: data bundle want = %v, get = %v", i, expected, data)
}
}
}
func TestFrameworkDataRequest(t *testing.T) {
appName := "framework_test_datarequest"
etcdURLs := []string{"http://localhost:4001"}
// launch controller to setup etcd layout
ctl := controller.New(appName, etcd.NewClient(etcdURLs), 2, []string{"Parents", "Children"})
if err := ctl.InitEtcdLayout(); err != nil {
t.Fatalf("initEtcdLayout failed: %v", err)
}
defer ctl.DestroyEtcdLayout()
pDataChan := make(chan *tDataBundle, 1)
cDataChan := make(chan *tDataBundle, 1)
// simulate two tasks on two nodes -- 0 and 1
// 0 is parent, 1 is child
f0 := &framework{
name: appName,
etcdURLs: etcdURLs,
ln: createListener(t),
}
f1 := &framework{
name: appName,
etcdURLs: etcdURLs,
ln: createListener(t),
}
var wg sync.WaitGroup
taskBuilder := &testableTaskBuilder{
cDataChan: cDataChan,
pDataChan: pDataChan,
setupLatch: &wg,
}
f0.SetTaskBuilder(taskBuilder)
f0.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
f0.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
f1.SetTaskBuilder(taskBuilder)
f1.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
f1.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
taskBuilder.setupLatch.Add(2)
go f0.Start()
go f1.Start()
taskBuilder.setupLatch.Wait()
if f0.GetTaskID() != 0 {
f0, f1 = f1, f0
}
defer f0.ShutdownJob()
ctx := context.WithValue(context.Background(), epochKey, uint64(0))
f0.DataRequest(ctx, 1, "/proto.Regression/GetGradient", nil)
data := <-pDataChan
expected := &tDataBundle{
id: 1,
method: "/proto.Regression/GetGradient",
output: &pb.Gradient{1},
}
if !reflect.DeepEqual(data, expected) {
t.Errorf("data bundle want = %v, get = %v", expected, data)
}
f1.DataRequest(ctx, 0, "/proto.Regression/GetParameter", nil)
data = <-cDataChan
expected = &tDataBundle{
id: 0,
method: "/proto.Regression/GetParameter",
output: &pb.Parameter{1},
}
if !reflect.DeepEqual(data, expected) {
t.Errorf("data bundle want = %v, get = %v", expected, data)
}
}