// newWebsocketDialer returns a function that
// can be passed to utils/parallel.Try.Start.
func newWebsocketDialer(cfg *websocket.Config, opts DialOpts) func(<-chan struct{}) (io.Closer, error) {
openAttempt := utils.AttemptStrategy{
Total: opts.Timeout,
Delay: opts.RetryDelay,
}
return func(stop <-chan struct{}) (io.Closer, error) {
for a := openAttempt.Start(); a.Next(); {
select {
case <-stop:
return nil, parallel.ErrStopped
default:
}
logger.Infof("dialing %q", cfg.Location)
conn, err := websocket.DialConfig(cfg)
if err == nil {
return conn, nil
}
if a.HasNext() {
logger.Debugf("error dialing %q, will retry: %v", cfg.Location, err)
} else {
logger.Infof("error dialing %q: %v", cfg.Location, err)
return nil, fmt.Errorf("timed out connecting to %q", cfg.Location)
}
}
panic("unreachable")
}
}
func newServer() (*coretesting.MgoInstance, error) {
inst := &coretesting.MgoInstance{Params: []string{"--replSet", name}}
err := inst.Start(true)
if err != nil {
return nil, fmt.Errorf("Error starting mongo server: %s", err.Error())
}
// by dialing right now, we'll wait until it's running
strategy := utils.AttemptStrategy{Total: time.Second * 5, Delay: time.Millisecond * 100}
attempt := strategy.Start()
for attempt.Next() {
var session *mgo.Session
session, err = inst.DialDirect()
if err != nil {
err = fmt.Errorf("Error dialing mongo server %q: %s", inst.Addr(), err.Error())
} else {
session.SetMode(mgo.Monotonic, true)
err = session.Ping()
if err != nil {
err = fmt.Errorf("Error pinging mongo server %q: %s", inst.Addr(), err.Error())
}
session.Close()
}
if err == nil || !attempt.HasNext() {
break
}
}
return inst, err
}
func checkFileHasContents(c *gc.C, stor storage.StorageReader, name string, contents []byte, attempt utils.AttemptStrategy) {
r, err := storage.GetWithRetry(stor, name, attempt)
c.Assert(err, gc.IsNil)
c.Check(r, gc.NotNil)
defer r.Close()
data, err := ioutil.ReadAll(r)
c.Check(err, gc.IsNil)
c.Check(data, gc.DeepEquals, contents)
url, err := stor.URL(name)
c.Assert(err, gc.IsNil)
var resp *http.Response
for a := attempt.Start(); a.Next(); {
resp, err = utils.GetValidatingHTTPClient().Get(url)
c.Assert(err, gc.IsNil)
if resp.StatusCode != 404 {
break
}
c.Logf("get retrying after earlier get succeeded. *sigh*.")
}
c.Assert(err, gc.IsNil)
data, err = ioutil.ReadAll(resp.Body)
c.Assert(err, gc.IsNil)
defer resp.Body.Close()
c.Assert(resp.StatusCode, gc.Equals, 200, gc.Commentf("error response: %s", data))
c.Check(data, gc.DeepEquals, contents)
}
// ListWithRetry lists the files matching prefix from stor using the specified attempt strategy.
func ListWithRetry(stor StorageReader, prefix string, attempt utils.AttemptStrategy) (list []string, err error) {
for a := attempt.Start(); a.Next(); {
list, err = stor.List(prefix)
if err == nil || !stor.ShouldRetry(err) {
break
}
}
return list, err
}
// GetWithRetry gets the named file from stor using the specified attempt strategy.
func GetWithRetry(stor StorageReader, name string, attempt utils.AttemptStrategy) (r io.ReadCloser, err error) {
for a := attempt.Start(); a.Next(); {
r, err = stor.Get(name)
if err == nil || !stor.ShouldRetry(err) {
break
}
}
return r, err
}
func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
s.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
st, _ := s.OpenAPIAsNewMachine(c)
err := st.Ping()
c.Assert(err, gc.IsNil)
coretesting.MgoServer.Destroy()
attempt := utils.AttemptStrategy{
Total: coretesting.LongWait,
Delay: coretesting.ShortWait,
}
for a := attempt.Start(); a.Next(); {
if err := st.Ping(); err != nil {
c.Assert(err, gc.ErrorMatches, "connection is shut down")
return
}
}
c.Fatalf("timed out waiting for API server to die")
}
func (utilsSuite) TestAttemptTiming(c *gc.C) {
testAttempt := utils.AttemptStrategy{
Total: 0.25e9,
Delay: 0.1e9,
}
want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
t0 := time.Now()
for a := testAttempt.Start(); a.Next(); {
got = append(got, time.Now().Sub(t0))
}
got = append(got, time.Now().Sub(t0))
c.Assert(got, gc.HasLen, len(want))
const margin = 0.01e9
for i, got := range want {
lo := want[i] - margin
hi := want[i] + margin
if got < lo || got > hi {
c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
}
}
}
func ExampleAttempt_HasNext() {
// This example shows how Attempt.HasNext can be used to help
// structure an attempt loop. If the godoc example code allowed
// us to make the example return an error, we would uncomment
// the commented return statements.
attempts := utils.AttemptStrategy{
Total: 1 * time.Second,
Delay: 250 * time.Millisecond,
}
for attempt := attempts.Start(); attempt.Next(); {
x, err := doSomething()
if shouldRetry(err) && attempt.HasNext() {
continue
}
if err != nil {
// return err
return
}
doSomethingWith(x)
}
// return ErrTimedOut
return
}
func (s *MongoSuite) TestCurrentStatus(c *gc.C) {
session := root.MustDial()
defer session.Close()
inst1, err := newServer()
c.Assert(err, gc.IsNil)
defer inst1.Destroy()
defer Remove(session, inst1.Addr())
inst2, err := newServer()
c.Assert(err, gc.IsNil)
defer inst2.Destroy()
defer Remove(session, inst2.Addr())
strategy := utils.AttemptStrategy{Total: time.Second * 31, Delay: time.Millisecond * 100}
attempt := strategy.Start()
for attempt.Next() {
err = Add(session, Member{Address: inst1.Addr()}, Member{Address: inst2.Addr()})
if err == nil || !attempt.HasNext() {
break
}
}
c.Assert(err, gc.IsNil)
expected := &Status{
Name: name,
Members: []MemberStatus{{
Id: 1,
Address: root.Addr(),
Self: true,
ErrMsg: "",
Healthy: true,
State: PrimaryState,
}, {
Id: 2,
Address: inst1.Addr(),
Self: false,
ErrMsg: "",
Healthy: true,
State: SecondaryState,
}, {
Id: 3,
Address: inst2.Addr(),
Self: false,
ErrMsg: "",
Healthy: true,
State: SecondaryState,
}},
}
strategy.Total = time.Second * 90
attempt = strategy.Start()
var res *Status
for attempt.Next() {
var err error
res, err = CurrentStatus(session)
if err != nil {
if !attempt.HasNext() {
c.Errorf("Couldn't get status before timeout, got err: %v", err)
return
} else {
// try again
continue
}
}
if res.Members[0].State == PrimaryState &&
res.Members[1].State == SecondaryState &&
res.Members[2].State == SecondaryState {
break
}
if !attempt.HasNext() {
c.Errorf("Servers did not get into final state before timeout. Status: %#v", res)
return
}
}
for x, _ := range res.Members {
// non-empty uptime and ping
c.Check(res.Members[x].Uptime, gc.Not(gc.Equals), 0)
// ping is always going to be zero since we're on localhost
// so we can't really test it right now
// now overwrite Uptime so it won't throw off DeepEquals
res.Members[x].Uptime = 0
}
c.Check(res, jc.DeepEquals, expected)
}
func (s *MongoSuite) TestAddRemoveSet(c *gc.C) {
session := root.MustDial()
defer session.Close()
members := make([]Member, 0, 5)
// Add should be idempotent, so re-adding root here shouldn't result in
// two copies of root in the replica set
members = append(members, Member{Address: root.Addr(), Tags: initialTags})
instances := make([]*coretesting.MgoInstance, 0, 5)
instances = append(instances, root)
for x := 0; x < 4; x++ {
inst, err := newServer()
c.Assert(err, gc.IsNil)
instances = append(instances, inst)
defer inst.Destroy()
defer Remove(session, inst.Addr())
key := fmt.Sprintf("key%d", x)
val := fmt.Sprintf("val%d", x)
tags := map[string]string{key: val}
members = append(members, Member{Address: inst.Addr(), Tags: tags})
}
var err error
// We use a delay of 31s. Our Mongo Dial timeout is 15s, so this gives
// us 2 attempts before we give up.
strategy := utils.AttemptStrategy{Total: time.Second * 31, Delay: time.Millisecond * 100}
start := time.Now()
attemptCount := 0
attempt := strategy.Start()
for attempt.Next() {
attemptCount += 1
err = Add(session, members...)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting to Add got error: %v", err)
}
c.Logf("Add() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
expectedMembers := make([]Member, len(members))
for x, m := range members {
// Ids should start at 1 (for the root) and go up
m.Id = x + 1
expectedMembers[x] = m
}
var cfg *Config
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
cfg, err = CurrentConfig(session)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting CurrentConfig got error: %v", err)
}
c.Logf("CurrentConfig() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
c.Assert(cfg.Name, gc.Equals, name)
// 2 since we already changed it once
c.Assert(cfg.Version, gc.Equals, 2)
mems := cfg.Members
c.Assert(mems, jc.DeepEquals, expectedMembers)
// Now remove the last two Members
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
err = Remove(session, members[3].Address, members[4].Address)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting Remove got error: %v", err)
}
c.Logf("Remove() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
expectedMembers = expectedMembers[0:3]
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
mems, err = CurrentMembers(session)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting CurrentMembers got error: %v", err)
}
c.Logf("CurrentMembers() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
c.Assert(mems, jc.DeepEquals, expectedMembers)
// now let's mix it up and set the new members to a mix of the previous
// plus the new arbiter
mems = []Member{members[3], mems[2], mems[0], members[4]}
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
err = Set(session, mems)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting Set got error: %v", err)
c.Logf("current session mode: %v", session.Mode())
session.Refresh()
}
c.Logf("Set() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
// can dial whichever replica address here, mongo will figure it out
session = instances[0].MustDialDirect()
err = session.Ping()
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting session.Ping() got error: %v after %s", err, time.Since(start))
}
c.Logf("session.Ping() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(err, gc.IsNil)
expectedMembers = []Member{members[3], expectedMembers[2], expectedMembers[0], members[4]}
// any new members will get an id of max(other_ids...)+1
expectedMembers[0].Id = 4
expectedMembers[3].Id = 5
start = time.Now()
attemptCount = 0
attempt = strategy.Start()
for attempt.Next() {
attemptCount += 1
mems, err = CurrentMembers(session)
if err == nil || !attempt.HasNext() {
break
}
c.Logf("attempting CurrentMembers() got error: %v", err)
}
c.Assert(err, gc.IsNil)
c.Logf("CurrentMembers() %d attempts in %s", attemptCount, time.Since(start))
c.Assert(mems, jc.DeepEquals, expectedMembers)
}
func (utilsSuite) TestAttemptNextHasNext(c *gc.C) {
a := utils.AttemptStrategy{}.Start()
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.Next(), gc.Equals, false)
a = utils.AttemptStrategy{}.Start()
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.HasNext(), gc.Equals, false)
c.Assert(a.Next(), gc.Equals, false)
a = utils.AttemptStrategy{Total: 2e8}.Start()
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.HasNext(), gc.Equals, true)
time.Sleep(2e8)
c.Assert(a.HasNext(), gc.Equals, true)
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.Next(), gc.Equals, false)
a = utils.AttemptStrategy{Total: 1e8, Min: 2}.Start()
time.Sleep(1e8)
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.HasNext(), gc.Equals, true)
c.Assert(a.Next(), gc.Equals, true)
c.Assert(a.HasNext(), gc.Equals, false)
c.Assert(a.Next(), gc.Equals, false)
}
// startReplicaSet starts up a replica set with n mongo instances.
func startReplicaSet(n int) (_ []*testing.MgoInstance, err error) {
insts := make([]*testing.MgoInstance, 0, n)
root, err := newMongoInstance()
if err != nil {
return nil, err
}
insts = append(insts, root)
defer func() {
if err == nil {
return
}
for _, inst := range insts {
inst.Destroy()
}
}()
dialInfo := root.DialInfo()
dialInfo.Direct = true
dialInfo.Timeout = 60 * time.Second
session, err := root.DialDirect()
if err != nil {
return nil, fmt.Errorf("cannot dial root instance: %v", err)
}
defer session.Close()
logger.Infof("dialled root instance")
if err := replicaset.Initiate(session, root.Addr(), replicaSetName, nil); err != nil {
return nil, fmt.Errorf("cannot initiate replica set: %v", err)
}
var members []replicaset.Member
for i := 1; i < n; i++ {
inst, err := newMongoInstance()
if err != nil {
return nil, err
}
insts = append(insts, inst)
members = append(members, replicaset.Member{
Address: inst.Addr(),
Priority: newFloat64(0.1),
Id: i + 1,
})
}
attempt := utils.AttemptStrategy{
Total: 60 * time.Second,
Delay: 1 * time.Second,
}
for a := attempt.Start(); a.Next(); {
err := replicaset.Add(session, members...)
if err == nil {
break
}
logger.Errorf("cannot add members: %v", err)
if !a.HasNext() {
return nil, fmt.Errorf("timed out trying to add members")
}
logger.Errorf("retrying")
}
return insts, err
}
