func (c *Controller) goWithRetry(f func() error, errChan chan error, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() bo := backoff.NewExponentialBackOff() bo.InitialInterval = time.Millisecond * 250 bo.MaxInterval = time.Second * 1 bo.MaxElapsedTime = time.Minute * 2 // channel message can take some time ticker := backoff.NewTicker(bo) defer ticker.Stop() var err error for range ticker.C { if err = f(); err != nil { c.log.Error("err while operating: %s will retry...", err.Error()) continue } break } errChan <- err }() }
func (l *LifeCycle) newProcessFunc(callback func([]*ec2.Instance) error) func(body *string) error { return func(body *string) error { l.log.Debug("got event %s", *body) ticker := backoff.NewTicker(backoff.NewExponentialBackOff()) var res []*ec2.Instance var err error for _ = range ticker.C { if res, err = l.GetAutoScalingOperatingMachines(); err != nil { l.log.Error("Getting autoscaling operating IPs failed, will retry... err: %s", err.Error()) continue } if err = callback(res); err != nil { l.log.Error("Upserting records failed, will retry... err: %s", err.Error()) continue } ticker.Stop() break } return err } }
func backOffTick(initial, max time.Duration) *backoff.Ticker { b := backoff.NewExponentialBackOff() b.InitialInterval = initial b.MaxInterval = max b.MaxElapsedTime = 0 // infinite return backoff.NewTicker(b) }
func (b *Bongo) Connect() error { bo := backoff.NewExponentialBackOff() ticker := backoff.NewTicker(bo) defer ticker.Stop() var err error for _ = range ticker.C { if err = b.Broker.Connect(); err != nil { b.log.Error("err while connecting: %s will retry...", err.Error()) continue } break } if err != nil { return err } B = b b.log.Info("Bongo connected %t", true) // todo add gorm Connect() return nil }
func newTicker() *backoff.Ticker { b := backoff.NewExponentialBackOff() b.InitialInterval = 100 * time.Millisecond b.MaxInterval = 5 * time.Second b.MaxElapsedTime = 15 * time.Second return backoff.NewTicker(b) }
// Exec implements the Stage interface. func (r RetryStage) Exec(ctx context.Context, alerts ...*types.Alert) (context.Context, []*types.Alert, error) { var ( i = 0 b = backoff.NewExponentialBackOff() tick = backoff.NewTicker(b) iErr error ) defer tick.Stop() for { i++ // Always check the context first to not notify again. select { case <-ctx.Done(): if iErr != nil { return ctx, nil, iErr } return ctx, nil, ctx.Err() default: } select { case <-tick.C: if retry, err := r.integration.Notify(ctx, alerts...); err != nil { numFailedNotifications.WithLabelValues(r.integration.name).Inc() log.Debugf("Notify attempt %d failed: %s", i, err) if !retry { return ctx, alerts, fmt.Errorf("Cancelling notify retry due to unrecoverable error: %s", err) } // Save this error to be able to return the last seen error by an // integration upon context timeout. iErr = err } else { numNotifications.WithLabelValues(r.integration.name).Inc() return ctx, alerts, nil } case <-ctx.Done(): if iErr != nil { return ctx, nil, iErr } return ctx, nil, ctx.Err() } } }
// ExecWithExponentialBackoff executes an operation with an exponential backoff retry // and returns the operation's error func (c *Context) ExecWithExponentialBackoff(op backoff.Operation) error { var err error c.backOff.Reset() ticker := backoff.NewTicker(c.backOff) for range ticker.C { if err = op(); err != nil { continue } ticker.Stop() break } return err }
// MustPing pings a database with an exponential backoff. The // function panics if the database cannot be pinged after 15 minutes func MustPing(db *sql.DB) { var err error b := backoff.NewExponentialBackOff() ticker := backoff.NewTicker(b) // Ticks will continue to arrive when the previous operation is still running, // so operations that take a while to fail could run in quick succession. for _ = range ticker.C { if err = db.Ping(); err != nil { logger.Info("pinging database...", err.Error()) continue } ticker.Stop() return } panic("Could not ping database!") }
func Save(d *schemas.Developer) error { if d.Salt == "" { d.Salt = uuid.New() d.Password = util.HashPassword(d.Password, d.Salt) } var err error b := backoff.NewTicker(backoff.NewExponentialBackOff()).C for _ = range b { if err = devs.Insert(d); err != nil { continue } break } return err }
func (p *PubNub) grantAccess(s *pubnub.AuthSettings) error { bo := backoff.NewExponentialBackOff() bo.MaxElapsedTime = MaxRetryDuration ticker := backoff.NewTicker(bo) var err error tryCount := 0 for range ticker.C { if err = p.grant.Grant(s); err != nil { tryCount++ p.log.Error("Could not grant access: %s will retry... (%d time(s))", err, tryCount) continue } ticker.Stop() } return err }
func (p *PubNub) publish(c ChannelManager, message interface{}) error { bo := backoff.NewExponentialBackOff() bo.MaxElapsedTime = MaxRetryDuration ticker := backoff.NewTicker(bo) defer ticker.Stop() var err error tryCount := 0 for range ticker.C { if err = p.pub.Push(c.PrepareName(), message); err != nil { tryCount++ p.log.Error("Could not publish message: %s will retry... (%d time(s))", err, tryCount) continue } ticker.Stop() } return err }
// Notify calls the underlying notifier with exponential backoff until it succeeds. // It aborts if the context is canceled or timed out. func (n *RetryNotifier) Notify(ctx context.Context, alerts ...*types.Alert) error { var ( i = 0 b = backoff.NewExponentialBackOff() tick = backoff.NewTicker(b) ) defer tick.Stop() for { i++ select { case <-tick.C: if err := n.notifier.Notify(ctx, alerts...); err != nil { log.Warnf("Notify attempt %d failed: %s", i, err) } else { return nil } case <-ctx.Done(): return ctx.Err() } } }
// validateClientWithBackoff repeatedly calls client.Ping until either // a successfull response or the context is canceled. func (c *influxdbCluster) validateClientWithBackoff(ctx context.Context) error { b := backoff.NewExponentialBackOff() b.MaxElapsedTime = c.startupTimeout ticker := backoff.NewTicker(b) defer ticker.Stop() done := ctx.Done() for { select { case <-done: return errors.New("canceled") case _, ok := <-ticker.C: if !ok { return errors.New("failed to connect to InfluxDB, retry limit reached") } _, _, err := c.client.Ping(ctx) if err != nil { c.logger.Println("D! failed to connect to InfluxDB, retrying... ", err) continue } return nil } } }
// waitForAction waits for a single action to finish. func waitForAction(ctx context.Context, client *godo.Client, action *godo.Action) error { if action == nil { return nil } // NOTE: if we need debugging, enable the following debug := false if debug { start := time.Now() log.Println("waiting for action to finish: ", action.ID) defer func() { log.Println("done: ", time.Since(start).Seconds()) }() } ticker := backoff.NewTicker(backoff.NewExponentialBackOff()) defer ticker.Stop() for { select { case <-ticker.C: var err error action, _, err = client.Actions.Get(action.ID) if err != nil { return err } if action.Status == actionErrored { return errors.New(action.String()) } if action.CompletedAt != nil || action.Status == actionCompleted { return nil } case <-ctx.Done(): return fmt.Errorf("timed out waiting for action %d to complete", action.ID) } } }
func (c *Client) vagetbackendauth(s url.URL, cid string) error { if cid == "" { cid = os.Getenv("VCLOUDAIR_COMPUTEID") } req := c.NewRequest(map[string]string{}, "POST", s, nil) // Add the Accept header for vCA req.Header.Add("Accept", "application/xml;version=5.6") // Set Authorization Header req.Header.Add("x-vchs-authorization", c.VAToken) // Adding exponential backoff to retry b := backoff.NewExponentialBackOff() b.MaxElapsedTime = time.Duration(30 * time.Second) ticker := backoff.NewTicker(b) var err error var resp *http.Response for t := range ticker.C { resp, err = checkResp(c.Http.Do(req)) if err != nil { fmt.Println(err, "retrying...", t) continue } ticker.Stop() break } if err != nil { return fmt.Errorf("error processing backend url action: %s", err) } defer resp.Body.Close() vcloudsession := new(vCloudSession) if err = decodeBody(resp, vcloudsession); err != nil { return fmt.Errorf("error decoding vcloudsession response: %s", err) } // Get the backend session information for _, s := range vcloudsession.VdcLink { if s.Name == cid { // Fetch the authorization token c.VCDToken = s.AuthorizationToken // Fetch the authorization header c.VCDAuthHeader = s.AuthorizationHeader u, err := url.ParseRequestURI(s.HREF) if err != nil { return fmt.Errorf("error decoding href: %s", err) } c.VCDVDCHREF = *u return nil } } return fmt.Errorf("error finding the right backend resource") }
func (s *Service) linkSubscriptions() error { s.logger.Println("I! linking subscriptions") b := backoff.NewExponentialBackOff() b.MaxElapsedTime = s.startupTimeout ticker := backoff.NewTicker(b) var err error var cli client.Client for range ticker.C { cli, err = s.NewClient() if err != nil { s.logger.Println("D! failed to connect to InfluxDB, retrying... ", err) continue } ticker.Stop() break } if err != nil { return err } numSubscriptions := int64(0) // Get all databases and retention policies var allSubs []subEntry resp, err := s.execQuery(cli, "SHOW DATABASES") if err != nil { return err } if len(resp.Results) == 1 && len(resp.Results[0].Series) == 1 && len(resp.Results[0].Series[0].Values) > 0 { dbs := resp.Results[0].Series[0].Values for _, v := range dbs { dbname := v[0].(string) rpResp, err := s.execQuery(cli, fmt.Sprintf(`SHOW RETENTION POLICIES ON "%s"`, dbname)) if err != nil { return err } if len(rpResp.Results) == 1 && len(rpResp.Results[0].Series) == 1 && len(rpResp.Results[0].Series[0].Values) > 0 { rps := rpResp.Results[0].Series[0].Values for _, v := range rps { rpname := v[0].(string) se := subEntry{ db: dbname, rp: rpname, name: s.clusterID, } allSubs = append(allSubs, se) } } } } // Get all existing subscriptions resp, err = s.execQuery(cli, "SHOW SUBSCRIPTIONS") if err != nil { return err } existingSubs := make(map[subEntry]subInfo) for _, res := range resp.Results { for _, series := range res.Series { for _, v := range series.Values { se := subEntry{ db: series.Name, } si := subInfo{} for i, c := range series.Columns { switch c { case "retention_policy": se.rp = v[i].(string) case "name": se.name = v[i].(string) case "mode": si.Mode = v[i].(string) case "destinations": destinations := v[i].([]interface{}) si.Destinations = make([]string, len(destinations)) for i := range destinations { si.Destinations[i] = destinations[i].(string) } } } if se.name == subName { // This is an old-style subscription, // drop it and recreate with new name. err := s.dropSub(cli, se.name, se.db, se.rp) if err != nil { return err } se.name = s.clusterID err = s.createSub(cli, se.name, se.db, se.rp, si.Mode, si.Destinations) if err != nil { return err } existingSubs[se] = si } if se.name == s.clusterID { existingSubs[se] = si } } } } // Compare to configured list startedSubs := make(map[subEntry]bool) all := len(s.configSubs) == 0 for se, si := range existingSubs { if (s.configSubs[se] || all) && !s.exConfigSubs[se] { // Check if this kapacitor instance is in the list of hosts for _, dest := range si.Destinations { u, err := url.Parse(dest) if err != nil { s.logger.Println("E! invalid URL in subscription destinations:", err) continue } pair := strings.Split(u.Host, ":") if pair[0] == s.hostname { numSubscriptions++ _, err := s.startListener(se.db, se.rp, *u) if err != nil { s.logger.Println("E! failed to start listener:", err) } startedSubs[se] = true break } } } } // create and start any new subscriptions for _, se := range allSubs { // If we have been configured to subscribe and the subscription is not started yet. if (s.configSubs[se] || all) && !startedSubs[se] && !s.exConfigSubs[se] { u, err := url.Parse("udp://:0") if err != nil { return fmt.Errorf("could not create valid destination url, is hostname correct? err: %s", err) } numSubscriptions++ addr, err := s.startListener(se.db, se.rp, *u) if err != nil { s.logger.Println("E! failed to start listener:", err) } // Get port from addr destination := fmt.Sprintf("udp://%s:%d", s.hostname, addr.Port) err = s.createSub(cli, se.name, se.db, se.rp, "ANY", []string{destination}) if err != nil { return err } } } kapacitor.NumSubscriptionsVar.Set(numSubscriptions) return nil }
func newTicker() *backoff.Ticker { b := backoff.NewExponentialBackOff() b.MaxInterval = 15 * time.Second return backoff.NewTicker(b) }
func (s *influxdbCluster) linkSubscriptions() error { s.logger.Println("D! linking subscriptions") b := backoff.NewExponentialBackOff() b.MaxElapsedTime = s.startupTimeout ticker := backoff.NewTicker(b) var err error var cli influxdb.Client for range ticker.C { cli, err = s.NewClient() if err != nil { s.logger.Println("D! failed to connect to InfluxDB, retrying... ", err) continue } ticker.Stop() break } if err != nil { return err } numSubscriptions := int64(0) tokens, err := s.AuthService.ListSubscriptionTokens() if err != nil { return errors.Wrap(err, "getting existing subscription tokens") } revokeTokens := make(map[string]bool, len(tokens)) for _, token := range tokens { revokeTokens[token] = true } // Get all databases and retention policies var allSubs []subEntry resp, err := s.execQuery(cli, &influxql.ShowDatabasesStatement{}) if err != nil { return err } if len(resp.Results) == 1 && len(resp.Results[0].Series) == 1 && len(resp.Results[0].Series[0].Values) > 0 { clusters := resp.Results[0].Series[0].Values for _, v := range clusters { clustername := v[0].(string) rpResp, err := s.execQuery(cli, &influxql.ShowRetentionPoliciesStatement{ Database: clustername, }) if err != nil { return err } if len(rpResp.Results) == 1 && len(rpResp.Results[0].Series) == 1 && len(rpResp.Results[0].Series[0].Values) > 0 { rps := rpResp.Results[0].Series[0].Values for _, v := range rps { rpname := v[0].(string) se := subEntry{ cluster: clustername, rp: rpname, name: s.subName, } allSubs = append(allSubs, se) } } } } // Get all existing subscriptions resp, err = s.execQuery(cli, &influxql.ShowSubscriptionsStatement{}) if err != nil { return err } existingSubs := make(map[subEntry]subInfo) for _, res := range resp.Results { for _, series := range res.Series { for _, v := range series.Values { se := subEntry{ cluster: series.Name, } si := subInfo{} for i, c := range series.Columns { switch c { case "retention_policy": se.rp = v[i].(string) case "name": se.name = v[i].(string) case "mode": si.Mode = v[i].(string) case "destinations": destinations := v[i].([]interface{}) si.Destinations = make([]string, len(destinations)) for i := range destinations { si.Destinations[i] = destinations[i].(string) } } } if se.name == legacySubName { // This is an old-style subscription, // drop it and recreate with new name. err := s.dropSub(cli, se.name, se.cluster, se.rp) if err != nil { return err } se.name = s.subName err = s.createSub(cli, se.name, se.cluster, se.rp, si.Mode, si.Destinations) if err != nil { return err } existingSubs[se] = si } else if se.name == s.clusterID { // This is an just the cluster ID // drop it and recreate with new name. err := s.dropSub(cli, se.name, se.cluster, se.rp) se.name = s.subName err = s.createSub(cli, se.name, se.cluster, se.rp, si.Mode, si.Destinations) if err != nil { return err } existingSubs[se] = si } else if se.name == s.subName { if len(si.Destinations) == 0 { s.logger.Println("E! found subscription without any destinations:", se) continue } u, err := url.Parse(si.Destinations[0]) if err != nil { s.logger.Println("E! found subscription with invalid destinations:", si) continue } host, port, err := net.SplitHostPort(u.Host) if err != nil { s.logger.Println("E! found subscription with invalid destinations:", si) continue } pn, err := strconv.ParseInt(port, 10, 64) if err != nil { s.logger.Println("E! found subscription with invalid destinations:", si) continue } // Check if the hostname, port or protocol have changed if host != s.hostname || u.Scheme != s.protocol || ((u.Scheme == "http" || u.Scheme == "https") && int(pn) != s.httpPort) || (s.useTokens && (u.User == nil || u.User.Username() != httpd.SubscriptionUser)) { // Remove access for changing subscriptions. if u.User != nil { if p, ok := u.User.Password(); ok { s.AuthService.RevokeSubscriptionAccess(p) } } // Something changed, drop the sub and let it get recreated s.dropSub(cli, se.name, se.cluster, se.rp) } else { existingSubs[se] = si // Do not revoke tokens that are still in use if u.User != nil { if t, ok := u.User.Password(); ok { revokeTokens[t] = false } } } } } } } // Compare to configured list startedSubs := make(map[subEntry]bool) all := len(s.configSubs) == 0 for se, si := range existingSubs { if (s.configSubs[se] || all) && !s.exConfigSubs[se] && !s.runningSubs[se] { // Check if this kapacitor instance is in the list of hosts for _, dest := range si.Destinations { u, err := url.Parse(dest) if err != nil { s.logger.Println("E! invalid URL in subscription destinations:", err) continue } host, port, err := net.SplitHostPort(u.Host) if host == s.hostname { numSubscriptions++ if u.Scheme == "udp" { _, err := s.startUDPListener(se.cluster, se.rp, port) if err != nil { s.logger.Println("E! failed to start UDP listener:", err) } } startedSubs[se] = true s.runningSubs[se] = true break } } } } // create and start any new subscriptions for _, se := range allSubs { // If we have been configured to subscribe and the subscription is not started yet. if (s.configSubs[se] || all) && !startedSubs[se] && !s.exConfigSubs[se] && !s.runningSubs[se] { var destination string switch s.protocol { case "http", "https": if s.useTokens { // Generate token token, err := s.generateRandomToken() if err != nil { return errors.Wrap(err, "generating token") } err = s.AuthService.GrantSubscriptionAccess(token, se.cluster, se.rp) if err != nil { return err } u := url.URL{ Scheme: s.protocol, User: url.UserPassword(httpd.SubscriptionUser, token), Host: fmt.Sprintf("%s:%d", s.hostname, s.httpPort), } destination = u.String() } else { u := url.URL{ Scheme: s.protocol, Host: fmt.Sprintf("%s:%d", s.hostname, s.httpPort), } destination = u.String() } case "udp": addr, err := s.startUDPListener(se.cluster, se.rp, "0") if err != nil { s.logger.Println("E! failed to start UDP listener:", err) } destination = fmt.Sprintf("udp://%s:%d", s.hostname, addr.Port) } numSubscriptions++ err = s.createSub(cli, se.name, se.cluster, se.rp, "ANY", []string{destination}) if err != nil { return err } s.runningSubs[se] = true } } // revoke any extra tokens for t, revoke := range revokeTokens { if revoke { s.AuthService.RevokeSubscriptionAccess(t) } } kapacitor.NumSubscriptionsVar.Set(numSubscriptions) return nil }