func upgradeMongoWatcher(st *state.State, stopch <-chan struct{}, machineID string, maybeStopMongo StopMongo) error {
m, err := st.Machine(machineID)
if err != nil {
return errors.Annotatef(err, "cannot start watcher for machine %q", machineID)
}
watch := m.Watch()
defer func() {
watch.Kill()
watch.Wait()
}()
for {
select {
case <-watch.Changes():
if err := m.Refresh(); err != nil {
return errors.Annotate(err, "cannot refresh machine information")
}
if !m.IsManager() {
continue
}
expectedVersion, err := m.StopMongoUntilVersion()
if err != nil {
return errors.Annotate(err, "cannot obtain minimum version of mongo")
}
if expectedVersion == mongo.Mongo24 {
continue
}
var isMaster bool
isMaster, err = mongo.IsMaster(st.MongoSession(), m)
if err != nil {
return errors.Annotatef(err, "cannot determine if machine %q is master", machineID)
}
err = maybeStopMongo(expectedVersion, isMaster)
if err != nil {
return errors.Annotate(err, "cannot determine if mongo must be stopped")
}
if !isMaster {
addrs := make([]string, len(m.Addresses()))
ssi, err := st.StateServingInfo()
if err != nil {
return errors.Annotate(err, "cannot obtain state serving info to stop mongo")
}
for i, addr := range m.Addresses() {
addrs[i] = net.JoinHostPort(addr.Value, strconv.Itoa(ssi.StatePort))
}
if err := replicaset.Remove(st.MongoSession(), addrs...); err != nil {
return errors.Annotatef(err, "cannot remove %q from replicaset", m.Id())
}
if err := m.SetStopMongoUntilVersion(mongo.Mongo24); err != nil {
return errors.Annotate(err, "cannot reset stop mongo flag")
}
}
case <-stopch:
return nil
}
}
}
// SetUpgradeMongoMode writes a value in the state server to be picked up
// by api servers to know that there is an upgrade ready to happen.
func (st *State) SetUpgradeMongoMode(v mongo.Version) (UpgradeMongoParams, error) {
currentInfo, err := st.ControllerInfo()
if err != nil {
return UpgradeMongoParams{}, errors.Annotate(err, "could not obtain current controller information")
}
result := UpgradeMongoParams{}
machines := []*Machine{}
for _, mID := range currentInfo.VotingMachineIds {
m, err := st.Machine(mID)
if err != nil {
return UpgradeMongoParams{}, errors.Annotate(err, "cannot change all the replicas")
}
isMaster, err := mongo.IsMaster(st.session, m)
if err != nil {
return UpgradeMongoParams{}, errors.Annotatef(err, "cannot determine if machine %q is master", mID)
}
paddr, err := m.PublicAddress()
if err != nil {
return UpgradeMongoParams{}, errors.Annotatef(err, "cannot obtain public address for machine: %v", m)
}
tag := m.Tag()
mtag := tag.(names.MachineTag)
member := HAMember{
Tag: mtag.Id(),
PublicAddress: paddr,
Series: m.Series(),
}
if isMaster {
result.Master = member
} else {
result.Members = append(result.Members, member)
}
machines = append(machines, m)
}
rsMembers, err := replicaset.CurrentMembers(st.session)
if err != nil {
return UpgradeMongoParams{}, errors.Annotate(err, "cannot obtain current replicaset members")
}
masterRs, err := replicaset.MasterHostPort(st.session)
if err != nil {
return UpgradeMongoParams{}, errors.Annotate(err, "cannot determine master on replicaset members")
}
for _, m := range rsMembers {
if m.Address != masterRs {
result.RsMembers = append(result.RsMembers, m)
}
}
for _, m := range machines {
if err := m.SetStopMongoUntilVersion(v); err != nil {
return UpgradeMongoParams{}, errors.Annotate(err, "cannot trigger replica shutdown")
}
}
return result, nil
}
func isMachineMaster(st *state.State, tag names.MachineTag) (bool, error) {
if st == nil {
// If there is no state, we aren't a master.
return false, nil
}
// Not calling the agent openState method as it does other checks
// we really don't care about here. All we need here is the machine
// so we can determine if we are the master or not.
machine, err := st.Machine(tag.Id())
if err != nil {
// This shouldn't happen, and if it does, the state worker will have
// found out before us, and already errored, or is likely to error out
// very shortly. All we do here is return the error. The state worker
// returns an error that will cause the agent to be terminated.
return false, errors.Trace(err)
}
isMaster, err := mongo.IsMaster(st.MongoSession(), machine)
if err != nil {
return false, errors.Trace(err)
}
return isMaster, nil
}
if st == nil {
// If there is no state, we aren't a master.
return false, nil
}
// Not calling the agent openState method as it does other checks
// we really don't care about here. All we need here is the machine
// so we can determine if we are the master or not.
machine, err := st.Machine(machineId)
if err != nil {
// This shouldn't happen, and if it does, the state worker will have
// found out before us, and already errored, or is likely to error out
// very shortly. All we do here is return the error. The state worker
// returns an error that will cause the agent to be terminated.
return false, errors.Trace(err)
}
isMaster, err := mongo.IsMaster(st.MongoSession(), machine)
if err != nil {
return false, errors.Trace(err)
}
return isMaster, nil
}
var getUpgradeRetryStrategy = func() utils.AttemptStrategy {
return utils.AttemptStrategy{
Delay: 2 * time.Minute,
Min: 5,
}
}
// jobsToTargets determines the upgrade targets corresponding to the
// jobs assigned to a machine agent. This determines the upgrade steps
func (c singularStateConn) IsMaster() (bool, error) {
return mongo.IsMaster(c.session, c.machine)
}