示例#1
0
文件: main.go 项目: bikong2/taskgraph
func main() {
	programType := flag.String("type", "", "(c) controller or (t) task")
	job := flag.String("job", "", "job name")
	etcdURLs := []string{"http://localhost:4001"}
	flag.Parse()

	if *job == "" {
		log.Fatalf("Please specify a job name")
	}

	ntask := uint64(2)
	switch *programType {
	case "c":
		log.Printf("controller")
		controller := controller.New(*job, etcd.NewClient(etcdURLs), ntask)
		controller.Start()
		controller.WaitForJobDone()
	case "t":
		log.Printf("task")
		bootstrap := framework.NewBootStrap(*job, etcdURLs, createListener(), nil)
		taskBuilder := &regression.SimpleTaskBuilder{
			GDataChan:          make(chan int32, 11),
			NumberOfIterations: 10,
			MasterConfig:       map[string]string{"writefile": "result.txt"},
		}
		bootstrap.SetTaskBuilder(taskBuilder)

		bootstrap.AddLinkage("Parents" : topo.NewTreeTopologyOfParent(2, ntask))
		bootstrap.AddLinkage("Children" : topo.NewTreeTopologyOfChildren(2, ntask))
		bootstrap.Start()
	default:
		log.Fatal("Please choose a type: (c) controller, (t) task")
	}
}
示例#2
0
// TestRequestDataEpochMismatch creates a scenario where data request happened
// with two different epochs. In this case, the server should back pressure and
// request client should get notified and return error.
func TestRequestDataEpochMismatch(t *testing.T) {
	t.Skip("TODO")
	job := "TestRequestDataEpochMismatch"
	etcdURLs := []string{"http://localhost:4001"}
	ctl := controller.New(job, etcd.NewClient(etcdURLs), 1, []string{"Parents", "Children"})
	ctl.InitEtcdLayout()
	defer ctl.DestroyEtcdLayout()

	fw := &framework{
		name:     job,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}
	var wg sync.WaitGroup
	fw.SetTaskBuilder(&testableTaskBuilder{
		setupLatch: &wg,
	})
	fw.AddLinkage("Parents", topo.NewTreeTopologyOfParent(1, 1))
	fw.AddLinkage("Children", topo.NewTreeTopologyOfChildren(1, 1))
	wg.Add(1)
	go fw.Start()
	wg.Wait()
	defer fw.ShutdownJob()

	addr, err := etcdutil.GetAddress(fw.etcdClient, job, fw.GetTaskID())
	if err != nil {
		t.Fatalf("GetAddress failed: %v", err)
	}
	addr = addr
	// _, err = frameworkhttp.RequestData(addr, "Parents", "req", 0, fw.GetTaskID(), 10, fw.GetLogger())
	// if err != frameworkhttp.ErrReqEpochMismatch {
	// 	t.Fatalf("error want = %v, but get = (%)", frameworkhttp.ErrReqEpochMismatch, err.Error())
	// }
}
示例#3
0
// This is used to show how to drive the network.
func driveWithTreeTopo(t *testing.T, jobName string, etcds []string, ntask uint64, taskBuilder taskgraph.TaskBuilder) {
	drive(
		t,
		jobName,
		etcds,
		taskBuilder,
		map[string]taskgraph.Topology{
			"Parents":  topo.NewTreeTopologyOfParent(2, ntask),
			"Children": topo.NewTreeTopologyOfChildren(2, ntask),
		},
	)
}
示例#4
0
// TestFrameworkFlagMetaReady and TestFrameworkDataRequest test basic workflows of
// framework impl. It uses a scenario with two nodes: 0 as parent, 1 as child.
// The basic idea is that when parent tries to talk to child and vice versa,
// there will be some data transferring and captured by application task.
// Here we have implemented a helper user task to capture those data, test if
// it's passed from framework correctly and unmodified.
func TestFrameworkFlagMeta(t *testing.T) {
	appName := "TestFrameworkFlagMeta"
	etcdURLs := []string{"http://localhost:4001"}
	// launch controller to setup etcd layout
	ctl := controller.New(appName, etcd.NewClient(etcdURLs), 2, []string{"Parents", "Children"})
	if err := ctl.InitEtcdLayout(); err != nil {
		t.Fatalf("initEtcdLayout failed: %v", err)
	}
	defer ctl.DestroyEtcdLayout()

	pDataChan := make(chan *tDataBundle, 1)
	cDataChan := make(chan *tDataBundle, 1)

	// simulate two tasks on two nodes -- 0 and 1
	// 0 is parent, 1 is child
	f0 := &framework{
		name:     appName,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}
	f1 := &framework{
		name:     appName,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}

	var wg sync.WaitGroup
	taskBuilder := &testableTaskBuilder{
		cDataChan:  cDataChan,
		pDataChan:  pDataChan,
		setupLatch: &wg,
	}
	f0.SetTaskBuilder(taskBuilder)
	f0.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
	f0.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
	f1.SetTaskBuilder(taskBuilder)
	f1.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
	f1.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))

	taskBuilder.setupLatch.Add(2)
	go f0.Start()
	go f1.Start()
	taskBuilder.setupLatch.Wait()
	if f0.GetTaskID() != 0 {
		f0, f1 = f1, f0
	}

	defer f0.ShutdownJob()

	tests := []struct {
		cMeta string
		pMeta string
	}{
		{"parent", "child"},
		{"ParamReady", "GradientReady"},
	}

	ctx := context.WithValue(context.Background(), epochKey, uint64(0))
	for i, tt := range tests {
		// 0: F#FlagChildMetaReady -> 1: T#ParentMetaReady
		f0.FlagMeta(ctx, "Children", tt.cMeta)
		// from child(1)'s view
		data := <-cDataChan
		expected := &tDataBundle{id: 0, meta: tt.cMeta}
		if !reflect.DeepEqual(data, expected) {
			t.Errorf("#%d: data bundle want = %v, get = %v", i, expected, data)
		}

		// 1: F#FlagParentMetaReady -> 0: T#ChildMetaReady
		f1.FlagMeta(ctx, "Parents", tt.pMeta)
		// from parent(0)'s view
		data = <-pDataChan
		expected = &tDataBundle{id: 1, meta: tt.pMeta}
		if !reflect.DeepEqual(data, expected) {
			t.Errorf("#%d: data bundle want = %v, get = %v", i, expected, data)
		}
	}
}
示例#5
0
func TestFrameworkDataRequest(t *testing.T) {
	appName := "framework_test_datarequest"
	etcdURLs := []string{"http://localhost:4001"}
	// launch controller to setup etcd layout
	ctl := controller.New(appName, etcd.NewClient(etcdURLs), 2, []string{"Parents", "Children"})
	if err := ctl.InitEtcdLayout(); err != nil {
		t.Fatalf("initEtcdLayout failed: %v", err)
	}
	defer ctl.DestroyEtcdLayout()

	pDataChan := make(chan *tDataBundle, 1)
	cDataChan := make(chan *tDataBundle, 1)
	// simulate two tasks on two nodes -- 0 and 1
	// 0 is parent, 1 is child
	f0 := &framework{
		name:     appName,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}
	f1 := &framework{
		name:     appName,
		etcdURLs: etcdURLs,
		ln:       createListener(t),
	}

	var wg sync.WaitGroup
	taskBuilder := &testableTaskBuilder{
		cDataChan:  cDataChan,
		pDataChan:  pDataChan,
		setupLatch: &wg,
	}
	f0.SetTaskBuilder(taskBuilder)
	f0.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
	f0.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))
	f1.SetTaskBuilder(taskBuilder)
	f1.AddLinkage("Parents", topo.NewTreeTopologyOfParent(2, 2))
	f1.AddLinkage("Children", topo.NewTreeTopologyOfChildren(2, 2))

	taskBuilder.setupLatch.Add(2)
	go f0.Start()
	go f1.Start()
	taskBuilder.setupLatch.Wait()
	if f0.GetTaskID() != 0 {
		f0, f1 = f1, f0
	}

	defer f0.ShutdownJob()
	ctx := context.WithValue(context.Background(), epochKey, uint64(0))

	f0.DataRequest(ctx, 1, "/proto.Regression/GetGradient", nil)
	data := <-pDataChan
	expected := &tDataBundle{
		id:     1,
		method: "/proto.Regression/GetGradient",
		output: &pb.Gradient{1},
	}
	if !reflect.DeepEqual(data, expected) {
		t.Errorf("data bundle want = %v, get = %v", expected, data)
	}
	f1.DataRequest(ctx, 0, "/proto.Regression/GetParameter", nil)
	data = <-cDataChan
	expected = &tDataBundle{
		id:     0,
		method: "/proto.Regression/GetParameter",
		output: &pb.Parameter{1},
	}
	if !reflect.DeepEqual(data, expected) {
		t.Errorf("data bundle want = %v, get = %v", expected, data)
	}
}