func (mr *Machiner) loop() error {
m, err := mr.st.Machine(mr.id)
if state.IsNotFound(err) {
return worker.ErrDead
} else if err != nil {
return err
}
w := m.Watch()
defer watcher.Stop(w, &mr.tomb)
for {
select {
case <-mr.tomb.Dying():
return tomb.ErrDying
case <-w.Changes():
if err := m.Refresh(); state.IsNotFound(err) {
return worker.ErrDead
} else if err != nil {
return err
}
if m.Life() != state.Alive {
// If the machine is Dying, it has no units,
// and can be safely set to Dead.
if err := m.EnsureDead(); err != nil {
return err
}
return worker.ErrDead
}
}
}
panic("unreachable")
}
func (s *RelationSuite) TestRetrieveNotFound(c *C) {
subway := state.Endpoint{"subway", "mongodb", "db", state.RoleRequirer, charm.ScopeGlobal}
mongo := state.Endpoint{"mongo", "mongodb", "server", state.RoleProvider, charm.ScopeGlobal}
_, err := s.State.EndpointsRelation(subway, mongo)
c.Assert(err, ErrorMatches, `relation "subway:db mongo:server" not found`)
c.Assert(state.IsNotFound(err), Equals, true)
_, err = s.State.Relation(999)
c.Assert(err, ErrorMatches, `relation 999 not found`)
c.Assert(state.IsNotFound(err), Equals, true)
}
func (s *MachineSuite) TestHostUnits(c *C) {
m, conf := s.primeAgent(c, state.JobHostUnits)
a := s.newAgent(c, m)
mgr, reset := patchDeployManager(c, conf.StateInfo, conf.DataDir)
defer reset()
go func() { c.Check(a.Run(nil), IsNil) }()
defer func() { c.Check(a.Stop(), IsNil) }()
svc, err := s.State.AddService("wordpress", s.AddTestingCharm(c, "wordpress"))
c.Assert(err, IsNil)
u0, err := svc.AddUnit()
c.Assert(err, IsNil)
u1, err := svc.AddUnit()
c.Assert(err, IsNil)
mgr.waitDeployed(c)
err = u0.AssignToMachine(m)
c.Assert(err, IsNil)
mgr.waitDeployed(c, u0.Name())
err = u0.EnsureDying()
c.Assert(err, IsNil)
mgr.waitDeployed(c, u0.Name())
err = u1.AssignToMachine(m)
c.Assert(err, IsNil)
mgr.waitDeployed(c, u0.Name(), u1.Name())
err = u0.EnsureDead()
c.Assert(err, IsNil)
mgr.waitDeployed(c, u1.Name())
err = u0.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
// watchLoop watches the service's exposed flag for changes.
func (sd *serviceData) watchLoop(exposed bool) {
defer sd.tomb.Done()
w := sd.service.Watch()
defer watcher.Stop(w, &sd.tomb)
for {
select {
case <-sd.tomb.Dying():
return
case _, ok := <-w.Changes():
if !ok {
sd.fw.tomb.Kill(watcher.MustErr(w))
return
}
if err := sd.service.Refresh(); err != nil {
if !state.IsNotFound(err) {
sd.fw.tomb.Kill(err)
}
return
}
change := sd.service.IsExposed()
if change == exposed {
continue
}
exposed = change
select {
case sd.fw.exposedChange <- &exposedChange{sd, change}:
case <-sd.tomb.Dying():
return
}
}
}
}
// startMachine creates a new data value for tracking details of the
// machine and starts watching the machine for units added or removed.
func (fw *Firewaller) startMachine(id string) error {
machined := &machineData{
fw: fw,
id: id,
unitds: make(map[string]*unitData),
ports: make([]state.Port, 0),
}
m, err := machined.machine()
if state.IsNotFound(err) {
return nil
} else if err != nil {
return fmt.Errorf("worker/firewaller: cannot watch machine units: %v", err)
}
unitw := m.WatchUnits()
select {
case <-fw.tomb.Dying():
stop("units watcher", unitw)
return tomb.ErrDying
case change, ok := <-unitw.Changes():
if !ok {
stop("units watcher", unitw)
return watcher.MustErr(unitw)
}
fw.machineds[id] = machined
err = fw.unitsChanged(&unitsChange{machined, change})
if err != nil {
stop("units watcher", unitw)
return fmt.Errorf("worker/firewaller: start watching machine %d faild: %v", id, err)
}
}
go machined.watchLoop(unitw)
return nil
}
// restoreRelations reconciles the supplied relation state dirs with the
// remote state of the corresponding relations.
func (u *Uniter) restoreRelations() error {
dirs, err := relation.ReadAllStateDirs(u.relationsDir)
if err != nil {
return err
}
for id, dir := range dirs {
remove := false
rel, err := u.st.Relation(id)
if state.IsNotFound(err) {
remove = true
} else if err != nil {
return err
}
if err = u.addRelation(rel, dir); err == state.ErrCannotEnterScope {
remove = true
} else if err != nil {
return err
}
if remove {
// If the previous execution was interrupted in the process of
// joining or departing the relation, the directory will be empty
// and the state is sane.
if err := dir.Remove(); err != nil {
return fmt.Errorf("cannot synchronize relation state: %v", err)
}
}
}
return nil
}
// watchLoop watches the unit for port changes.
func (ud *unitData) watchLoop(latestPorts []state.Port) {
defer ud.tomb.Done()
w := ud.unit.Watch()
defer watcher.Stop(w, &ud.tomb)
for {
select {
case <-ud.tomb.Dying():
return
case _, ok := <-w.Changes():
if !ok {
ud.fw.tomb.Kill(watcher.MustErr(w))
return
}
if err := ud.unit.Refresh(); err != nil {
if !state.IsNotFound(err) {
ud.fw.tomb.Kill(err)
}
return
}
change := ud.unit.OpenedPorts()
if samePorts(change, latestPorts) {
continue
}
latestPorts = append(latestPorts[:0], change...)
select {
case ud.fw.portsChange <- &portsChange{ud, change}:
case <-ud.tomb.Dying():
return
}
}
}
}
// NextTools returns the next changed tools, waiting
// until the tools are actually set.
func (w *toolsWaiter) NextTools(c *C) (*state.Tools, error) {
for _ = range w.changes {
err := w.tooler.Refresh()
if err != nil {
return nil, fmt.Errorf("cannot refresh: %v", err)
}
if w.tooler.Life() == state.Dead {
return nil, fmt.Errorf("object is dead")
}
tools, err := w.tooler.AgentTools()
if state.IsNotFound(err) {
c.Logf("tools not yet set")
continue
}
if err != nil {
return nil, err
}
changed := w.lastTools == nil || *tools != *w.lastTools
w.lastTools = tools
if changed {
return tools, nil
}
c.Logf("found same tools")
}
return nil, fmt.Errorf("watcher closed prematurely: %v", w.watcher.Err())
}
func (s *MachineSuite) TestMachineRefresh(c *C) {
m0, err := s.State.AddMachine(state.JobHostUnits)
c.Assert(err, IsNil)
oldId, _ := m0.InstanceId()
m1, err := s.State.Machine(m0.Id())
c.Assert(err, IsNil)
err = m0.SetInstanceId("umbrella/0")
c.Assert(err, IsNil)
newId, _ := m0.InstanceId()
m1Id, _ := m1.InstanceId()
c.Assert(m1Id, Equals, oldId)
err = m1.Refresh()
c.Assert(err, IsNil)
m1Id, _ = m1.InstanceId()
c.Assert(m1Id, Equals, newId)
err = m0.EnsureDead()
c.Assert(err, IsNil)
err = s.State.RemoveMachine(m0.Id())
c.Assert(err, IsNil)
err = m0.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
func (s *UnitSuite) TestRefresh(c *C) {
unit1, err := s.State.Unit(s.unit.Name())
c.Assert(err, IsNil)
err = s.unit.SetPrivateAddress("example.local")
c.Assert(err, IsNil)
err = s.unit.SetPublicAddress("example.foobar.com")
c.Assert(err, IsNil)
address, ok := unit1.PrivateAddress()
c.Assert(ok, Equals, false)
address, ok = unit1.PublicAddress()
c.Assert(ok, Equals, false)
err = unit1.Refresh()
c.Assert(err, IsNil)
address, ok = unit1.PrivateAddress()
c.Assert(ok, Equals, true)
c.Assert(address, Equals, "example.local")
address, ok = unit1.PublicAddress()
c.Assert(ok, Equals, true)
c.Assert(address, Equals, "example.foobar.com")
err = unit1.EnsureDead()
c.Assert(err, IsNil)
svc, err := s.State.Service(unit1.ServiceName())
c.Assert(err, IsNil)
err = svc.RemoveUnit(unit1)
c.Assert(err, IsNil)
err = unit1.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
// DestroyUnits removes the specified units from the state.
func (conn *Conn) DestroyUnits(names ...string) (err error) {
defer trivial.ErrorContextf(&err, "cannot destroy units")
var units []*state.Unit
for _, name := range names {
unit, err := conn.State.Unit(name)
switch {
case state.IsNotFound(err):
return fmt.Errorf("unit %q is not alive", name)
case err != nil:
return err
case unit.Life() != state.Alive:
return fmt.Errorf("unit %q is not alive", name)
case unit.IsPrincipal():
units = append(units, unit)
default:
return fmt.Errorf("unit %q is a subordinate", name)
}
}
for _, unit := range units {
if err := unit.EnsureDying(); err != nil {
return err
}
}
return nil
}
// unitChanged responds to changes in the unit.
func (f *filter) unitChanged() error {
if err := f.unit.Refresh(); err != nil {
if state.IsNotFound(err) {
return worker.ErrDead
}
return err
}
if f.life != f.unit.Life() {
switch f.life = f.unit.Life(); f.life {
case state.Dying:
log.Printf("worker/uniter: unit is dying")
close(f.outUnitDying)
f.outUpgrade = nil
case state.Dead:
log.Printf("worker/uniter: unit is dead")
return worker.ErrDead
}
}
if resolved := f.unit.Resolved(); resolved != f.resolved {
f.resolved = resolved
if f.resolved != state.ResolvedNone {
f.outResolved = f.outResolvedOn
}
}
return nil
}
func (s *ServiceSuite) TestLifeWithRemovableRelations(c *C) {
wordpress, err := s.State.AddService("wordpress", s.charm)
c.Assert(err, IsNil)
ep1 := state.Endpoint{"mysql", "ifce", "blah1", state.RoleProvider, charm.ScopeGlobal}
ep2 := state.Endpoint{"wordpress", "ifce", "blah1", state.RoleRequirer, charm.ScopeGlobal}
rel, err := s.State.AddRelation(ep1, ep2)
c.Assert(err, IsNil)
// Destroy a service with no units in relation scope; check service and
// unit removed.
err = wordpress.Destroy()
c.Assert(err, IsNil)
err = wordpress.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
err = rel.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
func (s *UnitSuite) TestRunStop(c *C) {
unit, conf, _ := s.primeAgent(c)
a := s.newAgent(c, unit)
mgr, reset := patchDeployManager(c, conf.StateInfo, conf.DataDir)
defer reset()
go func() { c.Check(a.Run(nil), IsNil) }()
defer func() { c.Check(a.Stop(), IsNil) }()
timeout := time.After(5 * time.Second)
waitStarted:
for {
select {
case <-timeout:
c.Fatalf("no activity detected")
case <-time.After(50 * time.Millisecond):
err := unit.Refresh()
c.Assert(err, IsNil)
st, info, err := unit.Status()
c.Assert(err, IsNil)
switch st {
case state.UnitPending, state.UnitInstalled:
c.Logf("waiting...")
continue
case state.UnitStarted:
c.Logf("started!")
break waitStarted
case state.UnitDown:
s.State.StartSync()
c.Logf("unit is still down")
default:
c.Fatalf("unexpected status %s %s", st, info)
}
}
}
// Check no subordinates have been deployed.
mgr.waitDeployed(c)
// Add a relation with a subordinate service and wait for the subordinate
// to be deployed...
_, err := s.State.AddService("logging", s.AddTestingCharm(c, "logging"))
c.Assert(err, IsNil)
eps, err := s.State.InferEndpoints([]string{"wordpress", "logging"})
c.Assert(err, IsNil)
_, err = s.State.AddRelation(eps...)
c.Assert(err, IsNil)
mgr.waitDeployed(c, "logging/0")
// ...then kill the subordinate and wait for it to be recalled and removed.
logging0, err := s.State.Unit("logging/0")
c.Assert(err, IsNil)
err = logging0.EnsureDead()
c.Assert(err, IsNil)
mgr.waitDeployed(c)
err = logging0.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
func (s *MachineSuite) TestManageEnviron(c *C) {
m, _ := s.primeAgent(c, state.JobManageEnviron)
op := make(chan dummy.Operation, 200)
dummy.Listen(op)
a := s.newAgent(c, m)
done := make(chan error)
go func() {
done <- a.Run(nil)
}()
// Check that the provisioner and firewaller are alive by doing
// a rudimentary check that it responds to state changes.
// Add one unit to a service; it should get allocated a machine
// and then its ports should be opened.
charm := s.AddTestingCharm(c, "dummy")
svc, err := s.Conn.AddService("test-service", charm)
c.Assert(err, IsNil)
err = svc.SetExposed()
c.Assert(err, IsNil)
units, err := s.Conn.AddUnits(svc, 1)
c.Assert(err, IsNil)
c.Check(opRecvTimeout(c, s.State, op, dummy.OpStartInstance{}), NotNil)
// Wait for the instance id to show up in the state.
id1, err := units[0].AssignedMachineId()
c.Assert(err, IsNil)
m1, err := s.State.Machine(id1)
c.Assert(err, IsNil)
w := m1.Watch()
defer w.Stop()
for _ = range w.Changes() {
err = m1.Refresh()
c.Assert(err, IsNil)
_, err := m1.InstanceId()
if state.IsNotFound(err) {
continue
}
c.Assert(err, IsNil)
break
}
err = units[0].OpenPort("tcp", 999)
c.Assert(err, IsNil)
c.Check(opRecvTimeout(c, s.State, op, dummy.OpOpenPorts{}), NotNil)
err = a.Stop()
c.Assert(err, IsNil)
select {
case err := <-done:
c.Assert(err, IsNil)
case <-time.After(5 * time.Second):
c.Fatalf("timed out waiting for agent to terminate")
}
}
func isRemoved(st *state.State, name string) func(*C) bool {
return func(c *C) bool {
_, err := st.Unit(name)
if state.IsNotFound(err) {
return true
}
c.Assert(err, IsNil)
return false
}
}
func (s addSubordinateRelation) step(c *C, ctx *context) {
if _, err := ctx.st.Service("logging"); state.IsNotFound(err) {
_, err := ctx.st.AddService("logging", ctx.s.AddTestingCharm(c, "logging"))
c.Assert(err, IsNil)
}
eps, err := ctx.st.InferEndpoints([]string{"logging", "u:" + s.ifce})
c.Assert(err, IsNil)
_, err = ctx.st.AddRelation(eps...)
c.Assert(err, IsNil)
}
func (s *ServiceSuite) TestReadUnitWithChangingState(c *C) {
// Check that reading a unit after removing the service
// fails nicely.
err := s.service.Destroy()
c.Assert(err, IsNil)
err = s.service.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
_, err = s.State.Unit("mysql/0")
c.Assert(err, ErrorMatches, `unit "mysql/0" not found`)
}
func (s *RelationSuite) TestDestroy(c *C) {
// Add a relation, and check we can only do so once.
riak, err := s.State.AddService("riak", s.AddTestingCharm(c, "riak"))
c.Assert(err, IsNil)
riakEP, err := riak.Endpoint("ring")
c.Assert(err, IsNil)
rel, err := s.State.AddRelation(riakEP)
c.Assert(err, IsNil)
err = rel.Destroy()
c.Assert(err, IsNil)
_, err = s.State.Relation(rel.Id())
c.Assert(state.IsNotFound(err), Equals, true)
_, err = s.State.EndpointsRelation(riakEP)
c.Assert(state.IsNotFound(err), Equals, true)
rels, err := riak.Relations()
c.Assert(err, IsNil)
c.Assert(rels, HasLen, 0)
}
func (s relationState) step(c *C, ctx *context) {
err := ctx.relation.Refresh()
if s.removed {
c.Assert(state.IsNotFound(err), Equals, true)
return
}
c.Assert(err, IsNil)
c.Assert(ctx.relation.Life(), Equals, s.life)
}
// waitRemoved waits for the supplied machine to be removed from state.
func (s *ProvisionerSuite) waitRemoved(c *C, m *state.Machine) {
s.waitMachine(c, m, func() bool {
err := m.Refresh()
if state.IsNotFound(err) {
return true
}
c.Assert(err, IsNil)
c.Logf("machine %v is still %s", m, m.Life())
return false
})
}
func (s *ServiceSuite) TestLifeWithUnits(c *C) {
unit, err := s.service.AddUnit()
c.Assert(err, IsNil)
err = s.service.Destroy()
c.Assert(err, IsNil)
err = unit.EnsureDead()
c.Assert(err, IsNil)
err = s.service.RemoveUnit(unit)
c.Assert(err, IsNil)
err = s.service.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
// unitsChanged responds to changes to the assigned units.
func (fw *Firewaller) unitsChanged(change *unitsChange) error {
changed := []*unitData{}
for _, name := range change.units {
unit, err := fw.st.Unit(name)
if err != nil && !state.IsNotFound(err) {
return err
}
var machineId string
if unit != nil {
machineId, err = unit.AssignedMachineId()
if state.IsNotFound(err) {
continue
} else if err != nil {
if _, ok := err.(*state.NotAssignedError); !ok {
return err
}
}
}
if unitd, known := fw.unitds[name]; known {
knownMachineId := fw.unitds[name].machined.id
if unit == nil || unit.Life() == state.Dead || machineId != knownMachineId {
fw.forgetUnit(unitd)
changed = append(changed, unitd)
log.Debugf("worker/firewaller: stopped watching unit %s", name)
}
} else if unit != nil && unit.Life() != state.Dead && fw.machineds[machineId] != nil {
err = fw.startUnit(unit, machineId)
if err != nil {
return err
}
changed = append(changed, fw.unitds[name])
log.Debugf("worker/firewaller: started watching unit %s", name)
}
}
if err := fw.flushUnits(changed); err != nil {
return fmt.Errorf("cannot change firewall ports: %v", err)
}
return nil
}
func (s *ServiceSuite) TestLifeWithReferencedRelations(c *C) {
wordpress, err := s.State.AddService("wordpress", s.charm)
c.Assert(err, IsNil)
ep1 := state.Endpoint{"mysql", "ifce", "blah1", state.RoleProvider, charm.ScopeGlobal}
ep2 := state.Endpoint{"wordpress", "ifce", "blah1", state.RoleRequirer, charm.ScopeGlobal}
rel, err := s.State.AddRelation(ep1, ep2)
c.Assert(err, IsNil)
// Join a unit to the wordpress side to keep the relation alive.
unit, err := wordpress.AddUnit()
c.Assert(err, IsNil)
ru, err := rel.Unit(unit)
c.Assert(err, IsNil)
err = ru.EnterScope(nil)
c.Assert(err, IsNil)
// Set Dying, and check that the relation also becomes Dying.
err = s.service.Destroy()
c.Assert(err, IsNil)
err = rel.Refresh()
c.Assert(err, IsNil)
c.Assert(rel.Life(), Equals, state.Dying)
// Check that no new relations can be added.
ep3 := state.Endpoint{"mysql", "ifce", "blah2", state.RolePeer, charm.ScopeGlobal}
_, err = s.State.AddRelation(ep3)
c.Assert(err, ErrorMatches, `cannot add relation "mysql:blah2": service "mysql" is not alive`)
// Leave scope on the counterpart side; check the service and relation
// are both removed.
err = ru.LeaveScope()
c.Assert(err, IsNil)
err = s.service.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
err = rel.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}
// reconcileInstances compares the initially started watcher for machines,
// units and services with the opened and closed ports of the instances and
// opens and closes the appropriate ports for each instance.
func (fw *Firewaller) reconcileInstances() error {
for _, machined := range fw.machineds {
m, err := machined.machine()
if state.IsNotFound(err) {
if err := fw.forgetMachine(machined); err != nil {
return err
}
continue
} else if err != nil {
return err
}
instanceId, err := m.InstanceId()
if err != nil {
return err
}
instances, err := fw.environ.Instances([]state.InstanceId{instanceId})
if err == environs.ErrNoInstances {
return nil
} else if err != nil {
return err
}
initialPorts, err := instances[0].Ports(machined.id)
if err != nil {
return err
}
// Check which ports to open or to close.
toOpen := diff(machined.ports, initialPorts)
toClose := diff(initialPorts, machined.ports)
if len(toOpen) > 0 {
log.Printf("worker/firewaller: opening instance ports %v for machine %s",
toOpen, machined.id)
if err := instances[0].OpenPorts(machined.id, toOpen); err != nil {
// TODO(mue) Add local retry logic.
return err
}
state.SortPorts(toOpen)
}
if len(toClose) > 0 {
log.Printf("worker/firewaller: closing instance ports %v for machine %s",
toClose, machined.id)
if err := instances[0].ClosePorts(machined.id, toClose); err != nil {
// TODO(mue) Add local retry logic.
return err
}
state.SortPorts(toClose)
}
}
return nil
}
func (s *MachineSuite) TestMachineInstanceIdBlank(c *C) {
machine, err := s.State.AddMachine(state.JobHostUnits)
c.Assert(err, IsNil)
err = s.machines.Update(
D{{"_id", machine.Id()}},
D{{"$set", D{{"instanceid", ""}}}},
)
c.Assert(err, IsNil)
err = machine.Refresh()
c.Assert(err, IsNil)
iid, err := machine.InstanceId()
c.Assert(state.IsNotFound(err), Equals, true)
c.Assert(string(iid), Equals, "")
}
func (t *LiveTests) assertStartInstance(c *C, m *state.Machine) {
// Wait for machine to get an instance id.
for a := waitAgent.Start(); a.Next(); {
err := m.Refresh()
c.Assert(err, IsNil)
instId, err := m.InstanceId()
if state.IsNotFound(err) {
continue
}
c.Assert(err, IsNil)
_, err = t.Env.Instances([]state.InstanceId{instId})
c.Assert(err, IsNil)
return
}
c.Fatalf("provisioner failed to start machine after %v", waitAgent.Total)
}
// pendingOrDead looks up machines with ids and retuns those that do not
// have an instance id assigned yet, and also those that are dead.
func (p *Provisioner) pendingOrDead(ids []string) (pending, dead []*state.Machine, err error) {
// TODO(niemeyer): ms, err := st.Machines(alive)
for _, id := range ids {
m, err := p.st.Machine(id)
if state.IsNotFound(err) {
log.Infof("worker/provisioner: machine %q not found in state", m)
continue
}
if err != nil {
return nil, nil, err
}
switch m.Life() {
case state.Dying:
if _, ok := m.InstanceId(); ok {
continue
}
log.Infof("worker/provisioner: killing dying, unprovisioned machine %q", m)
if err := m.EnsureDead(); err != nil {
return nil, nil, err
}
fallthrough
case state.Dead:
dead = append(dead, m)
log.Infof("worker/provisioner: removing dead machine %q", m)
if err := m.Remove(); err != nil {
return nil, nil, err
}
continue
}
if instId, hasInstId := m.InstanceId(); !hasInstId {
status, _, err := m.Status()
if err != nil {
log.Infof("worker/provisioner: cannot get machine %q status: %v", m, err)
continue
}
if status == params.StatusPending {
pending = append(pending, m)
log.Infof("worker/provisioner: found machine %q pending provisioning", m)
continue
}
} else {
log.Infof("worker/provisioner: machine %v already started as instance %q", m, instId)
}
}
return
}
func (s waitSubordinateExists) step(c *C, ctx *context) {
timeout := time.After(5 * time.Second)
for {
ctx.st.StartSync()
select {
case <-timeout:
c.Fatalf("subordinate was not created")
case <-time.After(50 * time.Millisecond):
var err error
ctx.subordinate, err = ctx.st.Unit(s.name)
if state.IsNotFound(err) {
continue
}
c.Assert(err, IsNil)
return
}
}
}
func (s *RelationSuite) TestRefresh(c *C) {
riak, err := s.State.AddService("riak", s.AddTestingCharm(c, "riak"))
c.Assert(err, IsNil)
riakEP, err := riak.Endpoint("ring")
c.Assert(err, IsNil)
rel, err := s.State.AddRelation(riakEP)
c.Assert(err, IsNil)
rels, err := riak.Relations()
c.Assert(err, IsNil)
rel1 := rels[0]
err = rel.Destroy()
c.Assert(err, IsNil)
c.Assert(rel1.Life(), Equals, state.Alive)
err = rel1.Refresh()
c.Assert(state.IsNotFound(err), Equals, true)
}