// REST API instances that are started are killed when the tests end.
// When we eventually have a REST API Stop command this can be killed.
func startAPI() string {
// Start a REST API to talk to
rest.StreamingBufferWindow = 0.01
log.SetLevel(LOG_LEVEL)
r, _ := rest.New(rest.GetDefaultConfig())
c := control.New(control.GetDefaultConfig())
c.Start()
s := scheduler.New(scheduler.GetDefaultConfig())
s.SetMetricManager(c)
s.Start()
r.BindConfigManager(c.Config)
r.BindMetricManager(c)
r.BindTaskManager(s)
go func(ch <-chan error) {
// Block on the error channel. Will return exit status 1 for an error or just return if the channel closes.
err, ok := <-ch
if !ok {
return
}
log.Fatal(err)
}(r.Err())
r.SetAddress("127.0.0.1:0")
r.Start()
time.Sleep(100 * time.Millisecond)
return fmt.Sprintf("http://localhost:%d", r.Port())
}
func getDefaultMockConfig() *mockConfig {
return &mockConfig{
LogLevel: 3,
GoMaxProcs: 1,
LogPath: "",
Control: control.GetDefaultConfig(),
Scheduler: scheduler.GetDefaultConfig(),
RestAPI: GetDefaultConfig(),
}
}
// get the default snapd configuration
func getDefaultConfig() *Config {
return &Config{
LogLevel: defaultLogLevel,
GoMaxProcs: defaultGoMaxProcs,
LogPath: defaultLogPath,
Control: control.GetDefaultConfig(),
Scheduler: scheduler.GetDefaultConfig(),
RestAPI: rest.GetDefaultConfig(),
Tribe: tribe.GetDefaultConfig(),
}
}
func TestMockPluginLoad(t *testing.T) {
// These tests only work if SNAP_PATH is known.
// It is the responsibility of the testing framework to
// build the plugins first into the build dir.
Convey("make sure plugin has been built", t, func() {
err := helper.CheckPluginBuilt(SnapPath, PluginName)
So(err, ShouldBeNil)
Convey("ensure plugin loads and responds", func() {
c := control.New(control.GetDefaultConfig())
c.Start()
rp, _ := core.NewRequestedPlugin(PluginPath)
_, err := c.Load(rp)
So(err, ShouldBeNil)
})
})
}
func TestMockPluginLoad(t *testing.T) {
// These tests only work if SNAP_PATH is known.
// It is the responsibility of the testing framework to
// build the plugins first into the build dir.
if SnapPath != "" {
// Helper plugin trigger build if possible for this plugin
helper.BuildPlugin(PluginType, PluginName)
//
Convey("ensure plugin loads and responds", t, func() {
c := control.New(control.GetDefaultConfig())
c.Start()
rp, _ := core.NewRequestedPlugin(PluginPath)
_, err := c.Load(rp)
So(err, ShouldBeNil)
})
} else {
fmt.Printf("SNAP_PATH not set. Cannot test %s plugin.\n", PluginName)
}
}
func startTribes(count int) []int {
seed := ""
var wg sync.WaitGroup
var mgtPorts []int
for i := 0; i < count; i++ {
mgtPort := getPort()
mgtPorts = append(mgtPorts, mgtPort)
tribePort := getPort()
conf := tribe.GetDefaultConfig()
conf.Name = fmt.Sprintf("member-%v", mgtPort)
conf.BindAddr = "127.0.0.1"
conf.BindPort = tribePort
conf.Seed = seed
conf.RestAPIPort = mgtPort
conf.MemberlistConfig.PushPullInterval = 5 * time.Second
conf.MemberlistConfig.RetransmitMult = conf.MemberlistConfig.RetransmitMult * 2
if seed == "" {
seed = fmt.Sprintf("%s:%d", "127.0.0.1", tribePort)
}
t, err := tribe.New(conf)
if err != nil {
panic(err)
}
c := control.New(control.GetDefaultConfig())
c.RegisterEventHandler("tribe", t)
c.Start()
s := scheduler.New(scheduler.GetDefaultConfig())
s.SetMetricManager(c)
s.RegisterEventHandler("tribe", t)
s.Start()
t.SetPluginCatalog(c)
t.SetTaskManager(s)
t.Start()
r, _ := rest.New(rest.GetDefaultConfig())
r.BindMetricManager(c)
r.BindTaskManager(s)
r.BindTribeManager(t)
r.SetAddress("", mgtPort)
r.Start()
wg.Add(1)
timer := time.After(10 * time.Second)
go func(port int) {
defer wg.Done()
for {
select {
case <-timer:
panic("timed out")
default:
time.Sleep(100 * time.Millisecond)
resp := getMembers(port)
if resp.Meta.Code == 200 && len(resp.Body.(*rbody.TribeMemberList).Members) == count {
log.Infof("num of members %v", len(resp.Body.(*rbody.TribeMemberList).Members))
return
}
}
}
}(mgtPort)
}
wg.Wait()
uris := make([]int, len(mgtPorts))
for idx, port := range mgtPorts {
uris[idx] = port
}
return uris
}
func TestDistributedWorkflow(t *testing.T) {
Convey("Create a scheduler with 2 controls and load plugins", t, func() {
l, _ := net.Listen("tcp", ":0")
l.Close()
cfg := control.GetDefaultConfig()
cfg.ListenPort = l.Addr().(*net.TCPAddr).Port
c1 := control.New(cfg)
c1.Start()
m, _ := net.Listen("tcp", ":0")
m.Close()
cfg.ListenPort = m.Addr().(*net.TCPAddr).Port
port1 := cfg.ListenPort
c2 := control.New(cfg)
schcfg := GetDefaultConfig()
sch := New(schcfg)
c2.Start()
sch.SetMetricManager(c1)
err := sch.Start()
So(err, ShouldBeNil)
// Load appropriate plugins into each control.
mock2Path := helper.PluginFilePath("snap-plugin-collector-mock2")
passthruPath := helper.PluginFilePath("snap-plugin-processor-passthru")
filePath := helper.PluginFilePath("snap-plugin-publisher-mock-file")
// mock2 and file onto c1
rp, err := core.NewRequestedPlugin(mock2Path)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
rp, err = core.NewRequestedPlugin(filePath)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
// passthru on c2
rp, err = core.NewRequestedPlugin(passthruPath)
So(err, ShouldBeNil)
passthru, err := c2.Load(rp)
So(err, ShouldBeNil)
Convey("Test task with one local and one remote node", func() {
//Create a task
//Create a workflowmap
wf := dsWFMap(port1)
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
// stop the scheduler and control (since in nested Convey statements, the
// statements in the outer Convey execute for each of the inner Conveys
// independently; see https://github.com/smartystreets/goconvey/wiki/Execution-order
// for details on execution order in Convey)
sch.Stop()
c2.Stop()
})
Convey("Test task with invalid remote port", func() {
wf := dsWFMap(0)
controlproxy.MAX_CONNECTION_TIMEOUT = 1 * time.Second
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 1)
So(t, ShouldBeNil)
// stop the scheduler and control (since in nested Convey statements, the
// statements in the outer Convey execute for each of the inner Conveys
// independently; see https://github.com/smartystreets/goconvey/wiki/Execution-order
// for details on execution order in Convey)
sch.Stop()
c2.Stop()
})
Convey("Test task without remote plugin", func() {
_, err := c2.Unload(passthru)
So(err, ShouldBeNil)
wf := dsWFMap(port1)
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 1)
So(t, ShouldBeNil)
// stop the scheduler and control (since in nested Convey statements, the
// statements in the outer Convey execute for each of the inner Conveys
// independently; see https://github.com/smartystreets/goconvey/wiki/Execution-order
// for details on execution order in Convey)
sch.Stop()
c2.Stop()
})
Convey("Test task failing when control is stopped while task is running", func() {
wf := dsWFMap(port1)
// set timeout so that connection attempt through the controlproxy will fail after 1 second
controlproxy.MAX_CONNECTION_TIMEOUT = time.Second
// define an interval that the simple scheduler will run on every 100ms
interval := time.Millisecond * 100
// create our task; should be disabled after 3 failures
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(interval), wf, true)
// ensure task was created successfully
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
// create a channel to listen on for a response and setup an event handler
// that will respond on that channel once the 'TaskDisabledEvent' arrives
respChan := make(chan struct{})
sch.RegisterEventHandler("test", &failHandler{respChan})
// then stop the controller
c2.Stop()
// and wait for the response (with a 30 second timeout; just in case)
var ok bool
select {
case <-time.After(30 * time.Second):
// if get here, the select timed out waiting for a response; we don't
// expect to hit this timeout since it should only take 3 seconds for
// the workflow to fail to connect to the gRPC server three times, but
// it might if the task did not fail as expected
So("Timeout triggered waiting for disabled event", ShouldBeBlank)
case <-respChan:
// if get here, we got a response on the respChan
ok = true
}
So(ok, ShouldEqual, true)
// stop the scheduler (since in nested Convey statements, the
// statements in the outer Convey execute for each of the inner Conveys
// independently; see https://github.com/smartystreets/goconvey/wiki/Execution-order
// for details on execution order in Convey)
sch.Stop()
})
})
}
func TestDistributedSubscriptions(t *testing.T) {
Convey("Load control/scheduler with a mock remote scheduler", t, func() {
l, _ := net.Listen("tcp", ":0")
l.Close()
cfg := control.GetDefaultConfig()
cfg.ListenPort = l.Addr().(*net.TCPAddr).Port
c1 := control.New(cfg)
c1.Start()
m, _ := net.Listen("tcp", ":0")
m.Close()
cfg.ListenPort = m.Addr().(*net.TCPAddr).Port
port1 := cfg.ListenPort
c2 := control.New(cfg)
schcfg := GetDefaultConfig()
sch := New(schcfg)
c2.Start()
sch.SetMetricManager(c1)
err := sch.Start()
So(err, ShouldBeNil)
// Load appropriate plugins into each control.
mock2Path := helper.PluginFilePath("snap-plugin-collector-mock2")
passthruPath := helper.PluginFilePath("snap-plugin-processor-passthru")
filePath := helper.PluginFilePath("snap-plugin-publisher-mock-file")
// mock2 and file onto c1
rp, err := core.NewRequestedPlugin(mock2Path)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
rp, err = core.NewRequestedPlugin(filePath)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
// passthru on c2
rp, err = core.NewRequestedPlugin(passthruPath)
So(err, ShouldBeNil)
_, err = c2.Load(rp)
So(err, ShouldBeNil)
Convey("Starting task should not succeed if remote dep fails to subscribe", func() {
//Create a task
//Create a workflowmap
wf := dsWFMap(port1)
// Create a task that is not started immediately so we can
// validate deps correctly.
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, false)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
schTask := t.(*task)
remoteMockManager := &subscriptionManager{Fail: true}
schTask.RemoteManagers.Add(fmt.Sprintf("127.0.0.1:%v", port1), remoteMockManager)
localMockManager := &subscriptionManager{Fail: false}
schTask.RemoteManagers.Add("", localMockManager)
// Start task. We expect it to fail while subscribing deps
terrs := sch.StartTask(t.ID())
So(terrs, ShouldNotBeNil)
Convey("So dependencies should have been unsubscribed", func() {
// Ensure that unsubscribe call count is equal to subscribe call count
// i.e that every subscribe call was followed by an unsubscribe since
// we errored
So(remoteMockManager.UnsubscribeCallCount, ShouldEqual, remoteMockManager.SubscribeCallCount)
So(localMockManager.UnsubscribeCallCount, ShouldEqual, localMockManager.UnsubscribeCallCount)
})
})
Convey("Starting task should not succeed if missing local dep fails to subscribe", func() {
//Create a task
//Create a workflowmap
wf := dsWFMap(port1)
// Create a task that is not started immediately so we can
// validate deps correctly.
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, false)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
schTask := t.(*task)
localMockManager := &subscriptionManager{Fail: true}
schTask.RemoteManagers.Add("", localMockManager)
remoteMockManager := &subscriptionManager{Fail: false}
schTask.RemoteManagers.Add(fmt.Sprintf("127.0.0.1:%v", port1), remoteMockManager)
// Start task. We expect it to fail while subscribing deps
terrs := sch.StartTask(t.ID())
So(terrs, ShouldNotBeNil)
Convey("So dependencies should have been unsubscribed", func() {
// Ensure that unsubscribe call count is equal to subscribe call count
// i.e that every subscribe call was followed by an unsubscribe since
// we errored
So(remoteMockManager.UnsubscribeCallCount, ShouldEqual, remoteMockManager.SubscribeCallCount)
So(localMockManager.UnsubscribeCallCount, ShouldEqual, localMockManager.UnsubscribeCallCount)
})
})
Convey("Starting task should suceed if all deps are available", func() {
//Create a task
//Create a workflowmap
wf := dsWFMap(port1)
// Create a task that is not started immediately so we can
// validate deps correctly.
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, false)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
schTask := t.(*task)
localMockManager := &subscriptionManager{Fail: false}
schTask.RemoteManagers.Add("", localMockManager)
remoteMockManager := &subscriptionManager{Fail: false}
schTask.RemoteManagers.Add(fmt.Sprintf("127.0.0.1:%v", port1), remoteMockManager)
terrs := sch.StartTask(t.ID())
So(terrs, ShouldBeNil)
Convey("So all depndencies should have been subscribed to", func() {
// Ensure that unsubscribe call count is equal to subscribe call count
// i.e that every subscribe call was followed by an unsubscribe since
// we errored
So(localMockManager.SubscribeCallCount, ShouldBeGreaterThan, 0)
So(remoteMockManager.SubscribeCallCount, ShouldBeGreaterThan, 0)
})
})
})
}
// returns an array of the mgtports and the tribe port for the last node
func startTribes(count int, seed string) ([]int, int, *listenToSeedEvents) {
var wg sync.WaitGroup
var tribePort int
var mgtPorts []int
lpe := newListenToSeedEvents()
for i := 0; i < count; i++ {
mgtPort := getAvailablePort()
mgtPorts = append(mgtPorts, mgtPort)
tribePort = getAvailablePort()
conf := tribe.GetDefaultConfig()
conf.Name = fmt.Sprintf("member-%v", mgtPort)
conf.BindAddr = "127.0.0.1"
conf.BindPort = tribePort
conf.Seed = seed
conf.RestAPIPort = mgtPort
//conf.MemberlistConfig.PushPullInterval = 5 * time.Second
conf.MemberlistConfig.RetransmitMult = conf.MemberlistConfig.RetransmitMult * 2
t, err := tribe.New(conf)
if err != nil {
panic(err)
}
if seed == "" {
seed = fmt.Sprintf("%s:%d", "127.0.0.1", tribePort)
t.EventManager.RegisterHandler("tribe.tests", lpe)
}
cfg := control.GetDefaultConfig()
// get an available port to avoid conflicts (we aren't testing remote workflows here)
cfg.ListenPort = getAvailablePort()
c := control.New(cfg)
c.RegisterEventHandler("tribe", t)
c.Start()
s := scheduler.New(scheduler.GetDefaultConfig())
s.SetMetricManager(c)
s.RegisterEventHandler("tribe", t)
s.Start()
t.SetPluginCatalog(c)
t.SetTaskManager(s)
t.Start()
r, _ := New(GetDefaultConfig())
r.BindMetricManager(c)
r.BindTaskManager(s)
r.BindTribeManager(t)
r.SetAddress(fmt.Sprintf("127.0.0.1:%d", mgtPort))
r.Start()
wg.Add(1)
timer := time.After(10 * time.Second)
go func(port int) {
defer wg.Done()
for {
select {
case <-timer:
panic("timed out")
default:
time.Sleep(100 * time.Millisecond)
resp := getMembers(port)
if resp.Meta.Code == 200 && len(resp.Body.(*rbody.TribeMemberList).Members) >= count {
restLogger.Infof("num of members %v", len(resp.Body.(*rbody.TribeMemberList).Members))
return
}
}
}
}(mgtPort)
}
wg.Wait()
return mgtPorts, tribePort, lpe
}
func TestDistributedWorkflow(t *testing.T) {
Convey("Create a scheduler with 2 controls and load plugins", t, func() {
l, _ := net.Listen("tcp", ":0")
l.Close()
cfg := control.GetDefaultConfig()
cfg.ListenPort = l.Addr().(*net.TCPAddr).Port
c1 := control.New(cfg)
c1.Start()
m, _ := net.Listen("tcp", ":0")
m.Close()
cfg.ListenPort = m.Addr().(*net.TCPAddr).Port
port1 := cfg.ListenPort
c2 := control.New(cfg)
schcfg := GetDefaultConfig()
sch := New(schcfg)
c2.Start()
sch.SetMetricManager(c1)
err := sch.Start()
So(err, ShouldBeNil)
// Load appropriate plugins into each control.
mock2Path := path.Join(PluginPath, "snap-collector-mock2")
passthruPath := path.Join(PluginPath, "snap-processor-passthru")
filePath := path.Join(PluginPath, "snap-publisher-file")
// mock2 and file onto c1
rp, err := core.NewRequestedPlugin(mock2Path)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
rp, err = core.NewRequestedPlugin(filePath)
So(err, ShouldBeNil)
_, err = c1.Load(rp)
So(err, ShouldBeNil)
// passthru on c2
rp, err = core.NewRequestedPlugin(passthruPath)
So(err, ShouldBeNil)
passthru, err := c2.Load(rp)
So(err, ShouldBeNil)
Convey("Test task with one local and one remote node", func() {
//Create a task
//Create a workflowmap
wf := dsWFMap(port1)
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
})
Convey("Test task with invalid remote port", func() {
wf := dsWFMap(0)
controlproxy.MAX_CONNECTION_TIMEOUT = 1 * time.Second
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 1)
So(t, ShouldBeNil)
})
Convey("Test task without remote plugin", func() {
_, err := c2.Unload(passthru)
So(err, ShouldBeNil)
wf := dsWFMap(port1)
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(time.Second), wf, true)
So(len(errs.Errors()), ShouldEqual, 1)
So(t, ShouldBeNil)
})
Convey("Test task failing when control is stopped while task is running", func() {
wf := dsWFMap(port1)
controlproxy.MAX_CONNECTION_TIMEOUT = 10 * time.Second
interval := time.Millisecond * 100
t, errs := sch.CreateTask(schedule.NewSimpleSchedule(interval), wf, true)
So(len(errs.Errors()), ShouldEqual, 0)
So(t, ShouldNotBeNil)
c2.Stop()
// Give task time to fail
time.Sleep(time.Second)
tasks := sch.GetTasks()
var task core.Task
for _, v := range tasks {
task = v
}
So(task.State(), ShouldEqual, core.TaskDisabled)
})
})
}