func (a *UnitAgent) APIWorkers() (worker.Worker, error) {
agentConfig := a.CurrentConfig()
st, entity, err := openAPIState(agentConfig, a)
if err != nil {
return nil, err
}
dataDir := agentConfig.DataDir()
runner := worker.NewRunner(connectionIsFatal(st), moreImportant)
runner.StartWorker("upgrader", func() (worker.Worker, error) {
return upgrader.NewUpgrader(st.Upgrader(), agentConfig), nil
})
runner.StartWorker("logger", func() (worker.Worker, error) {
return workerlogger.NewLogger(st.Logger(), agentConfig), nil
})
runner.StartWorker("uniter", func() (worker.Worker, error) {
return uniter.NewUniter(st.Uniter(), entity.Tag(), dataDir), nil
})
runner.StartWorker("apiaddressupdater", func() (worker.Worker, error) {
return apiaddressupdater.NewAPIAddressUpdater(st.Uniter(), a), nil
})
runner.StartWorker("rsyslog", func() (worker.Worker, error) {
return newRsyslogConfigWorker(st.Rsyslog(), agentConfig, rsyslog.RsyslogModeForwarding)
})
return newCloseWorker(runner, st), nil
}
func (*runnerSuite) TestOneWorkerStartWhenStopping(c *gc.C) {
worker.RestartDelay = 3 * time.Second
runner := worker.NewRunner(allFatal, noImportance)
starter := newTestWorkerStarter()
starter.stopWait = make(chan struct{})
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
err = runner.StopWorker("id")
c.Assert(err, gc.IsNil)
err = runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
close(starter.stopWait)
starter.assertStarted(c, false)
// Check that the task is restarted immediately without
// the usual restart timeout delay.
t0 := time.Now()
starter.assertStarted(c, true)
restartDuration := time.Since(t0)
if restartDuration > 1*time.Second {
c.Fatalf("task did not restart immediately")
}
c.Assert(worker.Stop(runner), gc.IsNil)
}
func (a *agent) mongoWorker() (worker.Worker, error) {
dialInfo := testing.MgoDialInfo(a.hostPort)
session, err := mgo.DialWithInfo(dialInfo)
if err != nil {
return nil, err
}
mc := &mongoConn{
localHostPort: a.hostPort,
session: session,
}
runner := worker.NewRunner(
connectionIsFatal(mc),
func(err0, err1 error) bool { return true },
)
singularRunner, err := singular.New(runner, mc)
if err != nil {
return nil, fmt.Errorf("cannot start singular runner: %v", err)
}
a.notify.workerConnected()
singularRunner.StartWorker(fmt.Sprint("worker-", a.notify.id), func() (worker.Worker, error) {
return worker.NewSimpleWorker(func(stop <-chan struct{}) error {
return a.worker(session, stop)
}), nil
})
return runner, nil
}
func newRunner() worker.Runner {
return worker.NewRunner(
func(err error) bool {
return err == errFatal
},
func(err0, err1 error) bool { return true },
)
}
func (*runnerSuite) TestOneWorkerStartFatalError(c *gc.C) {
runner := worker.NewRunner(allFatal, noImportance)
starter := newTestWorkerStarter()
starter.startErr = errors.New("cannot start test task")
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
err = runner.Wait()
c.Assert(err, gc.Equals, starter.startErr)
}
func (*runnerSuite) TestOneWorkerStart(c *gc.C) {
runner := worker.NewRunner(noneFatal, noImportance)
starter := newTestWorkerStarter()
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
c.Assert(worker.Stop(runner), gc.IsNil)
starter.assertStarted(c, false)
}
func (*runnerSuite) TestOneWorkerDieFatalError(c *gc.C) {
runner := worker.NewRunner(allFatal, noImportance)
starter := newTestWorkerStarter()
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
dieErr := errors.New("error when running")
starter.die <- dieErr
err = runner.Wait()
c.Assert(err, gc.Equals, dieErr)
starter.assertStarted(c, false)
}
func (*runnerSuite) TestOneWorkerStopFatalError(c *gc.C) {
runner := worker.NewRunner(allFatal, noImportance)
starter := newTestWorkerStarter()
starter.stopErr = errors.New("stop error")
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
err = runner.StopWorker("id")
c.Assert(err, gc.IsNil)
err = runner.Wait()
c.Assert(err, gc.Equals, starter.stopErr)
}
// Init initializes the command for running.
func (a *UnitAgent) Init(args []string) error {
if a.UnitName == "" {
return requiredError("unit-name")
}
if !names.IsUnit(a.UnitName) {
return fmt.Errorf(`--unit-name option expects "<service>/<n>" argument`)
}
if err := a.AgentConf.CheckArgs(args); err != nil {
return err
}
a.runner = worker.NewRunner(isFatal, moreImportant)
return nil
}
func (*runnerSuite) TestOneWorkerRestartDelay(c *gc.C) {
worker.RestartDelay = 100 * time.Millisecond
runner := worker.NewRunner(noneFatal, noImportance)
starter := newTestWorkerStarter()
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
starter.die <- fmt.Errorf("non-fatal error")
starter.assertStarted(c, false)
t0 := time.Now()
starter.assertStarted(c, true)
restartDuration := time.Since(t0)
if restartDuration < worker.RestartDelay {
c.Fatalf("restart delay was not respected; got %v want %v", restartDuration, worker.RestartDelay)
}
}
func (*runnerSuite) TestOneWorkerRestart(c *gc.C) {
runner := worker.NewRunner(noneFatal, noImportance)
starter := newTestWorkerStarter()
err := runner.StartWorker("id", testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starter.assertStarted(c, true)
// Check it restarts a few times time.
for i := 0; i < 3; i++ {
starter.die <- fmt.Errorf("an error")
starter.assertStarted(c, false)
starter.assertStarted(c, true)
}
c.Assert(worker.Stop(runner), gc.IsNil)
starter.assertStarted(c, false)
}
func (*runnerSuite) TestErrorImportance(c *gc.C) {
moreImportant := func(err0, err1 error) bool {
return err0.(errorLevel) > err1.(errorLevel)
}
id := func(i int) string { return fmt.Sprint(i) }
runner := worker.NewRunner(allFatal, moreImportant)
for i := 0; i < 10; i++ {
starter := newTestWorkerStarter()
starter.stopErr = errorLevel(i)
err := runner.StartWorker(id(i), testWorkerStart(starter))
c.Assert(err, gc.IsNil)
}
err := runner.StopWorker(id(4))
c.Assert(err, gc.IsNil)
err = runner.Wait()
c.Assert(err, gc.Equals, errorLevel(9))
}
func (s *ContainerSetupSuite) setupContainerWorker(c *gc.C, tag string) {
runner := worker.NewRunner(allFatal, noImportance)
pr := s.st.Provisioner()
machine, err := pr.Machine(tag)
c.Assert(err, gc.IsNil)
err = machine.SetSupportedContainers(instance.ContainerTypes...)
c.Assert(err, gc.IsNil)
cfg := s.AgentConfigForTag(c, tag)
watcherName := fmt.Sprintf("%s-container-watcher", machine.Id())
handler := provisioner.NewContainerSetupHandler(runner, watcherName, instance.ContainerTypes, machine, pr, cfg)
runner.StartWorker(watcherName, func() (worker.Worker, error) {
return worker.NewStringsWorker(handler), nil
})
go func() {
runner.Wait()
}()
}
func (*runnerSuite) TestAllWorkersStoppedWhenOneDiesWithFatalError(c *gc.C) {
runner := worker.NewRunner(allFatal, noImportance)
var starters []*testWorkerStarter
for i := 0; i < 10; i++ {
starter := newTestWorkerStarter()
err := runner.StartWorker(fmt.Sprint(i), testWorkerStart(starter))
c.Assert(err, gc.IsNil)
starters = append(starters, starter)
}
for _, starter := range starters {
starter.assertStarted(c, true)
}
dieErr := errors.New("fatal error")
starters[4].die <- dieErr
err := runner.Wait()
c.Assert(err, gc.Equals, dieErr)
for _, starter := range starters {
starter.assertStarted(c, false)
}
}
func (*runnerSuite) TestFatalErrorWhileSelfStartWorker(c *gc.C) {
// Original deadlock problem that this tests for:
// A worker tries to call StartWorker in its start function
// at the same time another worker dies with a fatal error.
// It might not be able to send on startc.
runner := worker.NewRunner(allFatal, noImportance)
selfStarter := newTestWorkerStarter()
// make the startNotify channel synchronous so
// we can delay the start indefinitely.
selfStarter.startNotify = make(chan bool)
selfStarter.hook = func() {
runner.StartWorker("another", func() (worker.Worker, error) {
return nil, fmt.Errorf("no worker started")
})
}
err := runner.StartWorker("self starter", testWorkerStart(selfStarter))
c.Assert(err, gc.IsNil)
fatalStarter := newTestWorkerStarter()
fatalStarter.startErr = fmt.Errorf("a fatal error")
err = runner.StartWorker("fatal worker", testWorkerStart(fatalStarter))
c.Assert(err, gc.IsNil)
// Wait for the runner loop to react to the fatal
// error and go into final shutdown mode.
time.Sleep(10 * time.Millisecond)
// At this point, the loop is in shutdown mode, but the
// selfStarter's worker is still in its start function.
// When the start function continues (the first assertStarted
// allows that to happen) it will try to create a new
// worker. This failed in an earlier version of the code because the
// loop was not ready to receive start requests.
selfStarter.assertStarted(c, true)
selfStarter.assertStarted(c, false)
err = runner.Wait()
c.Assert(err, gc.Equals, fatalStarter.startErr)
}
func (*runnerSuite) TestFatalErrorWhileStarting(c *gc.C) {
// Original deadlock problem that this tests for:
// A worker dies with fatal error while another worker
// is inside start(). runWorker can't send startInfo on startedc.
runner := worker.NewRunner(allFatal, noImportance)
slowStarter := newTestWorkerStarter()
// make the startNotify channel synchronous so
// we can delay the start indefinitely.
slowStarter.startNotify = make(chan bool)
err := runner.StartWorker("slow starter", testWorkerStart(slowStarter))
c.Assert(err, gc.IsNil)
fatalStarter := newTestWorkerStarter()
fatalStarter.startErr = fmt.Errorf("a fatal error")
err = runner.StartWorker("fatal worker", testWorkerStart(fatalStarter))
c.Assert(err, gc.IsNil)
// Wait for the runner loop to react to the fatal
// error and go into final shutdown mode.
time.Sleep(10 * time.Millisecond)
// At this point, the loop is in shutdown mode, but the
// slowStarter's worker is still in its start function.
// When the start function continues (the first assertStarted
// allows that to happen) and returns the new Worker,
// runWorker will try to send it on runner.startedc.
// This test makes sure that succeeds ok.
slowStarter.assertStarted(c, true)
slowStarter.assertStarted(c, false)
err = runner.Wait()
c.Assert(err, gc.Equals, fatalStarter.startErr)
}
func (*runnerSuite) TestStopWorkerWhenDead(c *gc.C) {
runner := worker.NewRunner(allFatal, noImportance)
c.Assert(worker.Stop(runner), gc.IsNil)
c.Assert(runner.StopWorker("foo"), gc.Equals, worker.ErrDead)
}