Example #1
0
File: worker.go Project: bac/juju
// NewLock creates a gate.Lock to be used to synchronise workers which
// need to start after upgrades have completed. If no upgrade steps
// are required the Lock is unlocked and the version in agent's
// configuration is updated to the currently running version.
//
// The returned Lock should be passed to NewWorker.
func NewLock(a agent.Agent) (gate.Lock, error) {
	lock := gate.NewLock()

	if wrench.IsActive("machine-agent", "always-try-upgrade") {
		// Always enter upgrade mode. This allows test of upgrades
		// even when there's actually no upgrade steps to run.
		return lock, nil
	}

	err := a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		if !upgrades.AreUpgradesDefined(agentConfig.UpgradedToVersion()) {
			logger.Infof("no upgrade steps required or upgrade steps for %v "+
				"have already been run.", jujuversion.Current)
			lock.Unlock()

			// Even if no upgrade is required the version number in
			// the agent's config still needs to be bumped.
			agentConfig.SetUpgradedToVersion(jujuversion.Current)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return lock, nil
}
Example #2
0
func (s *wrenchSuite) TestSetEnabled(c *gc.C) {
	s.createWrenchDir(c)
	s.createWrenchFile(c, "foo", "bar")

	// Starts enabled.
	c.Assert(wrench.IsEnabled(), jc.IsTrue)
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsTrue)

	// Disable.
	c.Assert(wrench.SetEnabled(false), jc.IsTrue)
	c.Assert(wrench.IsEnabled(), jc.IsFalse)
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)

	// Enable again.
	c.Assert(wrench.SetEnabled(true), jc.IsFalse)
	c.Assert(wrench.IsEnabled(), jc.IsTrue)
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsTrue)
}
Example #3
0
func (s *wrenchSuite) TestDirectoryNotOwnedByJujuUser(c *gc.C) {
	s.createWrenchDir(c)
	s.tweakOwner(c, s.wrenchDir)

	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)

	c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{{
		loggo.ERROR,
		`wrench directory has incorrect ownership - wrench functionality disabled \(.+\)`,
	}})
}
Example #4
0
func (s *wrenchSuite) TestFileNotOwnedByJujuUser(c *gc.C) {
	s.createWrenchDir(c)
	filename := s.createWrenchFile(c, "foo", "bar")
	s.tweakOwner(c, filename)

	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)

	c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{{
		loggo.ERROR,
		`wrench file for foo/bar has incorrect ownership - ignoring ` + filename,
	}})
}
Example #5
0
func (c *upgradeWorkerContext) run(stop <-chan struct{}) error {
	if wrench.IsActive("machine-agent", "fail-upgrade-start") {
		return nil // Make the worker stop
	}

	select {
	case <-c.UpgradeComplete:
		// Our work is already done (we're probably being restarted
		// because the API connection has gone down), so do nothing.
		return nil
	default:
	}

	c.agentConfig = c.agent.CurrentConfig()

	// If the machine agent is a state server, flag that state
	// needs to be opened before running upgrade steps
	for _, job := range c.jobs {
		if job == params.JobManageEnviron {
			c.isStateServer = true
		}
	}
	// We need a *state.State for upgrades. We open it independently
	// of StateWorker, because we have no guarantees about when
	// and how often StateWorker might run.
	if c.isStateServer {
		var err error
		c.st, err = openStateForUpgrade(c.agent, c.agentConfig)
		if err != nil {
			return err
		}
		defer c.st.Close()
	}
	if err := c.runUpgrades(); err != nil {
		// Only return an error from the worker if the connection to
		// state went away (possible mongo master change). Returning
		// an error when the connection is lost will cause the agent
		// to restart.
		//
		// For other errors, the error is not returned because we want
		// the machine agent to stay running in an error state waiting
		// for user intervention.
		if isAPILostDuringUpgrade(err) {
			return err
		}
	} else {
		// Upgrade succeeded - signal that the upgrade is complete.
		close(c.UpgradeComplete)
	}
	return nil
}
Example #6
0
func getUpgradeStartTimeout(isMaster bool) time.Duration {
	if wrench.IsActive("machine-agent", "short-upgrade-timeout") {
		// This duration is fairly arbitrary. During manual testing it
		// avoids the normal long wait but still provides a small
		// window to check the environment status and logs before the
		// timeout is triggered.
		return time.Minute
	}

	if isMaster {
		return upgradeStartTimeoutMaster
	}
	return upgradeStartTimeoutSecondary
}
Example #7
0
func (s *wrenchSuite) TestFilePermsTooLoose(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("Windows is not fully POSIX compliant")
	}
	s.createWrenchDir(c)
	filename := s.createWrenchFile(c, "foo", "bar")
	err := os.Chmod(filename, 0666)
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)

	c.Assert(s.logWriter.Log(), jc.LogMatches, []jc.SimpleMessage{{
		loggo.ERROR,
		`wrench file for foo/bar should only be writable by owner - ignoring ` + filename,
	}})
}
Example #8
0
func (task *provisionerTask) stopInstances(instances []instance.Instance) error {
	// Although calling StopInstance with an empty slice should produce no change in the
	// provider, environs like dummy do not consider this a noop.
	if len(instances) == 0 {
		return nil
	}
	if wrench.IsActive("provisioner", "stop-instances") {
		return errors.New("wrench in the works")
	}

	ids := make([]instance.Id, len(instances))
	for i, inst := range instances {
		ids[i] = inst.Id()
	}
	if err := task.broker.StopInstances(ids...); err != nil {
		return errors.Annotate(err, "broker failed to stop instances")
	}
	return nil
}
Example #9
0
// InitialiseUsingAgent sets up a upgradeWorkerContext from a machine agent instance.
// It may update the agent's configuration.
func (c *upgradeWorkerContext) InitializeUsingAgent(a upgradingMachineAgent) error {
	if wrench.IsActive("machine-agent", "always-try-upgrade") {
		// Always enter upgrade mode. This allows test of upgrades
		// even when there's actually no upgrade steps to run.
		return nil
	}
	return a.ChangeConfig(func(agentConfig agent.ConfigSetter) error {
		if !upgrades.AreUpgradesDefined(agentConfig.UpgradedToVersion()) {
			logger.Infof("no upgrade steps required or upgrade steps for %v "+
				"have already been run.", version.Current.Number)
			close(c.UpgradeComplete)

			// Even if no upgrade is required the version number in
			// the agent's config still needs to be bumped.
			agentConfig.SetUpgradedToVersion(version.Current.Number)
		}
		return nil
	})
}
Example #10
0
// runUpgrades runs the upgrade operations for each job type and
// updates the updatedToVersion on success.
func (c *upgradeWorkerContext) runUpgrades() error {
	upgradeInfo, err := c.prepareForUpgrade()
	if err != nil {
		return err
	}

	if wrench.IsActive("machine-agent", "fail-upgrade") {
		return errors.New("wrench")
	}

	if err := c.agent.ChangeConfig(c.runUpgradeSteps); err != nil {
		return err
	}

	if err := c.finaliseUpgrade(upgradeInfo); err != nil {
		return err
	}
	return nil
}
Example #11
0
func (s *wrenchSuite) TestIsActiveMultiFeaturesWithMixedNewlines(c *gc.C) {
	s.createWrenchDir(c)
	s.createWrenchFile(c, "foo", "one\ntwo\r\nthree\nbar\n")
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsTrue)
	s.AssertActivationLogged(c)
}
Example #12
0
func (s *wrenchSuite) TestNoDirectory(c *gc.C) {
	s.PatchValue(wrench.WrenchDir, "/does/not/exist")
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)
	s.AssertDirErrorLogged(c)
}
Example #13
0
func (s *wrenchSuite) TestIsActiveWithWhitespace(c *gc.C) {
	s.createWrenchDir(c)
	s.createWrenchFile(c, "foo", "\tbar  ")
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsTrue)
	s.AssertActivationLogged(c)
}
Example #14
0
func (c *upgradeWorkerContext) run(stop <-chan struct{}) error {
	if wrench.IsActive("machine-agent", "fail-upgrade-start") {
		return nil // Make the worker stop
	}

	select {
	case <-c.UpgradeComplete:
		// Our work is already done (we're probably being restarted
		// because the API connection has gone down), so do nothing.
		return nil
	default:
	}

	c.agentConfig = c.agent.CurrentConfig()

	c.fromVersion = c.agentConfig.UpgradedToVersion()
	c.toVersion = version.Current.Number
	if c.fromVersion == c.toVersion {
		logger.Infof("upgrade to %v already completed.", c.toVersion)
		close(c.UpgradeComplete)
		return nil
	}

	if err := c.initTag(c.agentConfig.Tag()); err != nil {
		return errors.Trace(err)
	}

	// If the machine agent is a state server, flag that state
	// needs to be opened before running upgrade steps
	for _, job := range c.jobs {
		if job == multiwatcher.JobManageEnviron {
			c.isStateServer = true
		}
	}

	// We need a *state.State for upgrades. We open it independently
	// of StateWorker, because we have no guarantees about when
	// and how often StateWorker might run.
	if c.isStateServer {
		var err error
		if c.st, err = openStateForUpgrade(c.agent, c.agentConfig); err != nil {
			return err
		}
		defer c.st.Close()

		if c.isMaster, err = isMachineMaster(c.st, c.machineId); err != nil {
			return errors.Trace(err)
		}

		stor := storage.NewStorage(c.st.EnvironUUID(), c.st.MongoSession())
		registerSimplestreamsDataSource(stor)
	}
	if err := c.runUpgrades(); err != nil {
		// Only return an error from the worker if the connection to
		// state went away (possible mongo master change). Returning
		// an error when the connection is lost will cause the agent
		// to restart.
		//
		// For other errors, the error is not returned because we want
		// the machine agent to stay running in an error state waiting
		// for user intervention.
		if isAPILostDuringUpgrade(err) {
			return err
		}
		c.reportUpgradeFailure(err, false)

	} else {
		// Upgrade succeeded - signal that the upgrade is complete.
		logger.Infof("upgrade to %v completed successfully.", c.toVersion)
		c.agent.setMachineStatus(c.apiState, params.StatusStarted, "")
		close(c.UpgradeComplete)
	}
	return nil
}
Example #15
0
File: worker.go Project: bac/juju
func (w *upgradesteps) run() error {
	if wrench.IsActive("machine-agent", "fail-upgrade-start") {
		return nil // Make the worker stop
	}

	if w.upgradeComplete.IsUnlocked() {
		// Our work is already done (we're probably being restarted
		// because the API connection has gone down), so do nothing.
		return nil
	}

	w.fromVersion = w.agent.CurrentConfig().UpgradedToVersion()
	w.toVersion = jujuversion.Current
	if w.fromVersion == w.toVersion {
		logger.Infof("upgrade to %v already completed.", w.toVersion)
		w.upgradeComplete.Unlock()
		return nil
	}

	// If the machine agent is a controller, flag that state
	// needs to be opened before running upgrade steps
	for _, job := range w.jobs {
		if job == multiwatcher.JobManageModel {
			w.isController = true
		}
	}

	// We need a *state.State for upgrades. We open it independently
	// of StateWorker, because we have no guarantees about when
	// and how often StateWorker might run.
	if w.isController {
		var err error
		if w.st, err = w.openState(); err != nil {
			return err
		}
		defer w.st.Close()

		if w.isMaster, err = IsMachineMaster(w.st, w.tag.Id()); err != nil {
			return errors.Trace(err)
		}
	}

	if err := w.runUpgrades(); err != nil {
		// Only return an error from the worker if the connection to
		// state went away (possible mongo master change). Returning
		// an error when the connection is lost will cause the agent
		// to restart.
		//
		// For other errors, the error is not returned because we want
		// the machine agent to stay running in an error state waiting
		// for user intervention.
		if isAPILostDuringUpgrade(err) {
			return err
		}
		w.reportUpgradeFailure(err, false)

	} else {
		// Upgrade succeeded - signal that the upgrade is complete.
		logger.Infof("upgrade to %v completed successfully.", w.toVersion)
		w.machine.SetStatus(status.Started, "", nil)
		w.upgradeComplete.Unlock()
	}
	return nil
}
Example #16
0
func (s *wrenchSuite) TestMatchInOtherCategory(c *gc.C) {
	s.createWrenchDir(c)
	s.createWrenchFile(c, "other", "bar")
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)
	s.AssertFileErrorLogged(c)
}
Example #17
0
func (s *wrenchSuite) TestNoFile(c *gc.C) {
	s.createWrenchDir(c)
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)
	s.AssertFileErrorLogged(c)
}
Example #18
0
func (s *wrenchSuite) TestNotActive(c *gc.C) {
	s.createWrenchDir(c)
	s.createWrenchFile(c, "foo", "abc")
	c.Assert(wrench.IsActive("foo", "bar"), jc.IsFalse)
	s.AssertNothingLogged(c)
}
Example #19
0
	if err != nil {
		// This shouldn't happen, and if it does, the state worker will have
		// found out before us, and already errored, or is likely to error out
		// very shortly.  All we do here is return the error. The state worker
		// returns an error that will cause the agent to be terminated.
		return false, errors.Trace(err)
	}
	isMaster, err := mongo.IsMaster(st.MongoSession(), machine)
	if err != nil {
		return false, errors.Trace(err)
	}
	return isMaster, nil
}

var waitForOtherStateServers = func(st *state.State, isMaster bool) error {
	if wrench.IsActive("machine-agent", "fail-state-server-upgrade-wait") {
		return errors.New("failing other state servers check due to wrench")
	}
	// TODO(mjs) - for now, assume that the other state servers are
	// ready. This function will be fleshed out once the UpgradeInfo
	// work is done.
	return nil
}

var getUpgradeRetryStrategy = func() utils.AttemptStrategy {
	return utils.AttemptStrategy{
		Delay: 2 * time.Minute,
		Min:   5,
	}
}