Exemple #1
0
func mongoRestoreCall(runCommand utilsRun, tmpDir, mongoPath, adminPassword, migrationName string,
	dbs []string, statePort int, invalidSSL bool, batchSize int, callArgs retry.CallArgs) error {
	mongorestore := filepath.Join(mongoPath, "mongorestore")
	restoreParams := []string{
		"--ssl",
		"--port", strconv.Itoa(statePort),
		"--host", "localhost",
	}

	if invalidSSL {
		restoreParams = append(restoreParams, "--sslAllowInvalidCertificates")
	}
	if batchSize > 0 {
		restoreParams = append(restoreParams, "--batchSize", strconv.Itoa(batchSize))
	}
	if adminPassword != "" {
		restoreParams = append(restoreParams, "-u", "admin", "-p", adminPassword)
	}
	var out string
	if len(dbs) == 0 || dbs == nil {
		restoreParams = append(restoreParams, filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName)))
		restoreCallArgs := callArgs
		restoreCallArgs.Func = func() error {
			var err error
			out, err = runCommand(mongorestore, restoreParams...)
			if err == nil {
				return nil
			}
			logger.Errorf("cannot restore %v: %s", err, out)
			return err
		}
		if err := retry.Call(restoreCallArgs); err != nil {
			err := errors.Annotatef(err, "cannot restore dbs got: %s", out)
			logger.Errorf("%#v", err)
			return err
		}
	}
	for i := range dbs {
		restoreDbParams := append(restoreParams,
			fmt.Sprintf("--db=%s", dbs[i]),
			filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName), dbs[i]))
		restoreCallArgs := callArgs
		restoreCallArgs.Func = func() error {
			var err error
			out, err = runCommand(mongorestore, restoreDbParams...)
			if err == nil {
				return nil
			}
			logger.Errorf("cannot restore db %q: %v: got %s", dbs[i], err, out)
			return err
		}
		if err := retry.Call(restoreCallArgs); err != nil {
			return errors.Annotatef(err, "cannot restore db %q got: %s", dbs[i], out)
		}
		logger.Infof("Succesfully restored db %q", dbs[i])
	}
	return nil
}
Exemple #2
0
// deleteSecurityGroup attempts to delete the security group. Should it fail,
// the deletion is retried due to timing issues in openstack. A security group
// cannot be deleted while it is in use. Theoretically we terminate all the
// instances before we attempt to delete the associated security groups, but
// in practice nova hasn't always finished with the instance before it
// returns, so there is a race condition where we think the instance is
// terminated and hence attempt to delete the security groups but nova still
// has it around internally. To attempt to catch this timing issue, deletion
// of the groups is tried multiple times.
func deleteSecurityGroup(novaclient *nova.Client, name, id string) {
	logger.Debugf("deleting security group %q", name)
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			return novaclient.DeleteSecurityGroup(id)
		},
		NotifyFunc: func(err error, attempt int) {
			if attempt%4 == 0 {
				message := fmt.Sprintf("waiting to delete security group %q", name)
				if attempt != 4 {
					message = "still " + message
				}
				logger.Debugf(message)
			}
		},
		Attempts: 30,
		Delay:    time.Second,
		// TODO(dimitern): This should be fixed to take a clock.Clock arg, not
		// hard-coded WallClock, like in provider/ec2/securitygroups_test.go!
		// See PR juju:#5197, especially the code around autoAdvancingClock.
		// LP Bug: http://pad.lv/1580626.
		Clock: clock.WallClock,
	})
	if err != nil {
		logger.Warningf("cannot delete security group %q. Used by another model?", name)
	}
}
Exemple #3
0
// GetResource returns a reader for the resource's data.
func (client CSRetryClient) GetResource(req charmstore.ResourceRequest) (charmstore.ResourceData, error) {
	args := client.retryArgs // a copy

	var data charmstore.ResourceData
	args.Func = func() error {
		var err error
		data, err = client.Client.GetResource(req)
		if err != nil {
			return errors.Trace(err)
		}
		return nil
	}

	var lastErr error
	args.NotifyFunc = func(err error, i int) {
		// Remember the error we're hiding and then retry!
		logger.Debugf("(attempt %d) retrying resource download from charm store due to error: %v", i, err)
		lastErr = err
	}

	err := retry.Call(args)
	if retry.IsAttemptsExceeded(err) {
		return data, errors.Annotate(lastErr, "failed after retrying")
	}
	if err != nil {
		return data, errors.Trace(err)
	}

	return data, nil
}
Exemple #4
0
// dialAndLogin returns a mongo session logged in as a user with administrative
// privileges
func dialAndLogin(mongoInfo *mongo.MongoInfo, callArgs retry.CallArgs) (mgoSession, mgoDb, error) {
	var session *mgo.Session
	opts := mongo.DefaultDialOpts()
	callArgs.Func = func() error {
		// Try to connect, retry a few times until the db comes up.
		var err error
		session, err = mongo.DialWithInfo(mongoInfo.Info, opts)
		if err == nil {
			return nil
		}
		logger.Errorf("cannot open mongo connection: %v", err)
		return err
	}
	if err := retry.Call(callArgs); err != nil {
		return nil, nil, errors.Annotate(err, "error dialing mongo to resume HA")
	}
	admin := session.DB("admin")
	if mongoInfo.Tag != nil {
		if err := admin.Login(mongoInfo.Tag.String(), mongoInfo.Password); err != nil {
			return nil, nil, errors.Annotatef(err, "cannot log in to admin database as %q", mongoInfo.Tag)
		}
	} else if mongoInfo.Password != "" {
		if err := admin.Login(mongo.AdminUser, mongoInfo.Password); err != nil {
			return nil, nil, errors.Annotate(err, "cannot log in to admin database")
		}
	}
	return session, admin, nil
}
Exemple #5
0
func mongoDumpCall(
	runCommand utilsRun, tmpDir, mongoPath, adminPassword, migrationName string,
	statePort int, callArgs retry.CallArgs,
) (string, error) {
	mongodump := filepath.Join(mongoPath, "mongodump")
	dumpParams := []string{
		"--ssl",
		"-u", "admin",
		"-p", adminPassword,
		"--port", strconv.Itoa(statePort),
		"--host", "localhost",
		"--out", filepath.Join(tmpDir, fmt.Sprintf("migrateTo%sdump", migrationName)),
	}
	var out string
	callArgs.Func = func() error {
		var err error
		out, err = runCommand(mongodump, dumpParams...)
		if err == nil {
			return nil
		}
		logger.Errorf("cannot dump db %v: %s", err, out)
		return err
	}
	if err := retry.Call(callArgs); err != nil {
		logger.Errorf(out)
		return out, errors.Annotate(err, "cannot dump mongo db")
	}
	return out, nil
}
Exemple #6
0
// APICall places a call to the remote machine.
//
// This fills out the rpc.Request on the given facade, version for a given
// object id, and the specific RPC method. It marshalls the Arguments, and will
// unmarshall the result into the response object that is supplied.
func (s *state) APICall(facade string, version int, id, method string, args, response interface{}) error {
	retrySpec := retry.CallArgs{
		Func: func() error {
			return s.client.Call(rpc.Request{
				Type:    facade,
				Version: version,
				Id:      id,
				Action:  method,
			}, args, response)
		},
		IsFatalError: func(err error) bool {
			err = errors.Cause(err)
			ec, ok := err.(hasErrorCode)
			if !ok {
				return true
			}
			return ec.ErrorCode() != params.CodeRetry
		},
		Delay:       100 * time.Millisecond,
		MaxDelay:    1500 * time.Millisecond,
		MaxDuration: 10 * time.Second,
		BackoffFunc: retry.DoubleDelay,
		Clock:       s.clock,
	}
	err := retry.Call(retrySpec)
	return errors.Trace(err)
}
Exemple #7
0
func (*retrySuite) TestMissingFuncNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Attempts: 5,
		Delay:    time.Minute,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Func not valid`)
}
Exemple #8
0
func (*retrySuite) TestMissingDelayNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return errors.New("bah") },
		Attempts: 5,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Delay not valid`)
}
Exemple #9
0
func (*retrySuite) TestMissingAttemptsNotValid(c *gc.C) {
	err := retry.Call(retry.CallArgs{
		Func:  func() error { return errors.New("bah") },
		Delay: time.Minute,
		Clock: clock.WallClock,
	})
	c.Check(err, jc.Satisfies, errors.IsNotValid)
	c.Check(err, gc.ErrorMatches, `missing Attempts or MaxDuration not valid`)
}
Exemple #10
0
func (*retrySuite) TestSuccessHasNoDelay(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return nil },
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(clock.delays, gc.HasLen, 0)
}
Exemple #11
0
func (*retrySuite) TestAttemptsExceededError(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return funcErr },
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, gc.ErrorMatches, `attempt count exceeded: bah`)
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(retry.LastError(err), gc.Equals, funcErr)
}
Exemple #12
0
func (*retrySuite) TestFatalErrorsNotRetried(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:         func() error { return funcErr },
		IsFatalError: func(error) bool { return true },
		Attempts:     5,
		Delay:        time.Minute,
		Clock:        clock,
	})
	c.Assert(errors.Cause(err), gc.Equals, funcErr)
	c.Assert(clock.delays, gc.HasLen, 0)
}
Exemple #13
0
func (*retrySuite) TestBackoffErrors(c *gc.C) {
	// Backoff values of less than one are a validation error.
	for _, factor := range []float64{-2, 0.5} {
		err := retry.Call(retry.CallArgs{
			Func:          func() error { return errors.New("bah") },
			Attempts:      5,
			Delay:         time.Minute,
			BackoffFactor: factor,
		})
		c.Check(err, jc.Satisfies, errors.IsNotValid)
		c.Check(err, gc.ErrorMatches, `BackoffFactor of .* not valid`)
	}
}
Exemple #14
0
func (*retrySuite) TestWithWallClock(c *gc.C) {
	var attempts []int
	err := retry.Call(retry.CallArgs{
		Func: func() error { return errors.New("bah") },
		NotifyFunc: func(lastError error, attempt int) {
			attempts = append(attempts, attempt)
		},
		Attempts: 5,
		Delay:    time.Microsecond,
	})
	c.Assert(errors.Cause(err), jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(attempts, jc.DeepEquals, []int{1, 2, 3, 4, 5})
}
Exemple #15
0
// It appears that sometimes the lock is not cleared when we expect it to be.
// Capture and log any errors from the Unlock method and retry a few times.
func (s *store) unlock(lock *fslock.Lock) {
	err := retry.Call(retry.CallArgs{
		Func: lock.Unlock,
		NotifyFunc: func(err error, attempt int) {
			logger.Debugf("failed to unlock jujuclient lock: %s", err)
		},
		Attempts: 10,
		Delay:    50 * time.Millisecond,
		Clock:    clock.WallClock,
	})
	if err != nil {
		logger.Errorf("unable to unlock jujuclient lock: %s", err)
	}
}
Exemple #16
0
func (*retrySuite) TestAttempts(c *gc.C) {
	clock := &mockClock{}
	funcErr := errors.New("bah")
	err := retry.Call(retry.CallArgs{
		Func:     func() error { return funcErr },
		Attempts: 4,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	// We delay between attempts, and don't delay after the last one.
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute,
		time.Minute,
	})
}
Exemple #17
0
func (*retrySuite) TestMaxDuration(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Delay:       time.Minute,
		MaxDuration: 5 * time.Minute,
		Clock:       clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsDurationExceeded)
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute,
		time.Minute,
		time.Minute,
		time.Minute,
	})
}
Exemple #18
0
func (*retrySuite) TestBackoffFactor(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Clock:       clock,
		Attempts:    5,
		Delay:       time.Minute,
		BackoffFunc: retry.DoubleDelay,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		time.Minute * 2,
		time.Minute * 4,
		time.Minute * 8,
	})
}
Exemple #19
0
func (*retrySuite) TestMaxDurationDoubling(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Delay:       time.Minute,
		MaxDuration: 10 * time.Minute,
		BackoffFunc: retry.DoubleDelay,
		Clock:       clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsDurationExceeded)
	// Stops after seven minutes, because the next wait time
	// would take it to 15 minutes.
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		2 * time.Minute,
		4 * time.Minute,
	})
}
Exemple #20
0
// Dial returns a new connection to the MongoDB server.
func (inst *MgoInstance) Dial() (*mgo.Session, error) {
	var session *mgo.Session
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			var err error
			session, err = mgo.DialWithInfo(inst.DialInfo())
			return err
		},
		// Only interested in retrying the intermittent
		// 'unexpected message'.
		IsFatalError: func(err error) bool {
			return !strings.HasSuffix(err.Error(), "unexpected message")
		},
		Delay:    time.Millisecond,
		Clock:    clock.WallClock,
		Attempts: 5,
	})
	return session, err
}
Exemple #21
0
func (*retrySuite) TestCalledOnceEvenIfStopped(c *gc.C) {
	stop := make(chan struct{})
	clock := &mockClock{}
	called := false
	close(stop)
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			called = true
			return nil
		},
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
		Stop:     stop,
	})
	c.Assert(called, jc.IsTrue)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(clock.delays, gc.HasLen, 0)
}
Exemple #22
0
func (*retrySuite) TestStopChannel(c *gc.C) {
	clock := &mockClock{}
	stop := make(chan struct{})
	count := 0
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			if count == 2 {
				close(stop)
			}
			count++
			return errors.New("bah")
		},
		Attempts: 5,
		Delay:    time.Minute,
		Clock:    clock,
		Stop:     stop,
	})
	c.Assert(err, jc.Satisfies, retry.IsRetryStopped)
	c.Assert(clock.delays, gc.HasLen, 3)
}
Exemple #23
0
func (*retrySuite) TestMaxDelay(c *gc.C) {
	clock := &mockClock{}
	err := retry.Call(retry.CallArgs{
		Func:        func() error { return errors.New("bah") },
		Attempts:    7,
		Delay:       time.Minute,
		MaxDelay:    10 * time.Minute,
		BackoffFunc: retry.DoubleDelay,
		Clock:       clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(clock.delays, jc.DeepEquals, []time.Duration{
		time.Minute,
		2 * time.Minute,
		4 * time.Minute,
		8 * time.Minute,
		10 * time.Minute,
		10 * time.Minute,
	})
}
Exemple #24
0
func (*retrySuite) TestInfiniteRetries(c *gc.C) {
	// OK, we can't test infinite, but we'll go for lots.
	clock := &mockClock{}
	stop := make(chan struct{})
	count := 0
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			if count == 111 {
				close(stop)
			}
			count++
			return errors.New("bah")
		},
		Attempts: retry.UnlimitedAttempts,
		Delay:    time.Minute,
		Clock:    clock,
		Stop:     stop,
	})
	c.Assert(err, jc.Satisfies, retry.IsRetryStopped)
	c.Assert(clock.delays, gc.HasLen, count)
}
Exemple #25
0
// Reset resets the entire dummy environment and forgets any registered
// operation listener. All opened environments after Reset will share
// the same underlying state.
func Reset(c *gc.C) {
	logger.Infof("reset model")
	dummy.mu.Lock()
	dummy.ops = discardOperations
	oldState := dummy.state
	dummy.controllerState = nil
	dummy.state = make(map[string]*environState)
	dummy.newStatePolicy = stateenvirons.GetNewPolicyFunc(
		stateenvirons.GetNewEnvironFunc(environs.New),
	)
	dummy.supportsSpaces = true
	dummy.supportsSpaceDiscovery = false
	dummy.mu.Unlock()

	// NOTE(axw) we must destroy the old states without holding
	// the provider lock, or we risk deadlocking. Destroying
	// state involves closing the embedded API server, which
	// may require waiting on RPC calls that interact with the
	// EnvironProvider (e.g. EnvironProvider.Open).
	for _, s := range oldState {
		if s.apiListener != nil {
			s.apiListener.Close()
		}
		s.destroy()
	}
	if mongoAlive() {
		err := retry.Call(retry.CallArgs{
			Func: gitjujutesting.MgoServer.Reset,
			// Only interested in retrying the intermittent
			// 'unexpected message'.
			IsFatalError: func(err error) bool {
				return !strings.HasSuffix(err.Error(), "unexpected message")
			},
			Delay:    time.Millisecond,
			Clock:    clock.WallClock,
			Attempts: 5,
		})
		c.Assert(err, jc.ErrorIsNil)
	}
}
Exemple #26
0
// call will call the supplied function, with exponential backoff
// as long as the request returns an http.StatusTooManyRequests
// status.
func (c backoffAPIRequestCaller) call(f func() (autorest.Response, error)) error {
	var resp *http.Response
	return retry.Call(retry.CallArgs{
		Func: func() error {
			autorestResp, err := f()
			resp = autorestResp.Response
			return err
		},
		IsFatalError: func(err error) bool {
			return resp == nil || !autorest.ResponseHasStatusCode(resp, http.StatusTooManyRequests)
		},
		NotifyFunc: func(err error, attempt int) {
			logger.Debugf("attempt %d: %v", attempt, err)
		},
		Attempts:    -1,
		Delay:       retryDelay,
		MaxDelay:    maxRetryDelay,
		MaxDuration: maxRetryDuration,
		BackoffFunc: retry.DoubleDelay,
		Clock:       c.clock,
	})
}
Exemple #27
0
// deleteSecurityGroup attempts to delete the security group. Should it fail,
// the deletion is retried due to timing issues in openstack. A security group
// cannot be deleted while it is in use. Theoretically we terminate all the
// instances before we attempt to delete the associated security groups, but
// in practice nova hasn't always finished with the instance before it
// returns, so there is a race condition where we think the instance is
// terminated and hence attempt to delete the security groups but nova still
// has it around internally. To attempt to catch this timing issue, deletion
// of the groups is tried multiple times.
func deleteSecurityGroup(novaclient *nova.Client, name, id string) {
	logger.Debugf("deleting security group %q", name)
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			return novaclient.DeleteSecurityGroup(id)
		},
		NotifyFunc: func(err error, attempt int) {
			if attempt%4 == 0 {
				message := fmt.Sprintf("waiting to delete security group %q", name)
				if attempt != 4 {
					message = "still " + message
				}
				logger.Debugf(message)
			}
		},
		Attempts: 30,
		Delay:    time.Second,
		Clock:    clock.WallClock,
	})
	if err != nil {
		logger.Warningf("cannot delete security group %q. Used by another environment?", name)
	}
}
Exemple #28
0
func (*retrySuite) TestNotifyFunc(c *gc.C) {
	var (
		clock      = &mockClock{}
		funcErr    = errors.New("bah")
		attempts   []int
		funcErrors []error
	)
	err := retry.Call(retry.CallArgs{
		Func: func() error {
			return funcErr
		},
		NotifyFunc: func(lastError error, attempt int) {
			funcErrors = append(funcErrors, lastError)
			attempts = append(attempts, attempt)
		},
		Attempts: 3,
		Delay:    time.Minute,
		Clock:    clock,
	})
	c.Assert(err, jc.Satisfies, retry.IsAttemptsExceeded)
	c.Assert(clock.delays, gc.HasLen, 2)
	c.Assert(funcErrors, jc.DeepEquals, []error{funcErr, funcErr, funcErr})
	c.Assert(attempts, jc.DeepEquals, []int{1, 2, 3})
}