// doReconnect attempts a reconnection. It assumes that reconnectChan // and reconnectErrPtr are the same ones in c, but are passed in to // avoid having to take the mutex at the beginning of the method. func (c *Connection) doReconnect(ctx context.Context, reconnectChan chan struct{}, reconnectErrPtr *error) { // retry w/exponential backoff backoff.RetryNotify(func() error { // try to connect err := c.connect(ctx) select { case <-ctx.Done(): // context was canceled by Shutdown() or a user action *reconnectErrPtr = ctx.Err() // short-circuit Retry return nil default: } if dontRetryOnConnect(err) { // A fatal error happened. *reconnectErrPtr = err // short-circuit Retry return nil } return err }, backoff.NewExponentialBackOff(), // give the caller a chance to log any other error or adjust state c.handler.OnConnectError) // close the reconnect channel to signal we're connected. c.mutex.Lock() defer c.mutex.Unlock() close(reconnectChan) c.reconnectChan = nil c.cancelFunc = nil c.reconnectErrPtr = nil }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints []types.Constraint) error { config := api.DefaultConfig() config.Address = provider.Endpoint client, err := api.NewClient(config) if err != nil { return err } provider.client = client provider.Constraints = append(provider.Constraints, constraints...) pool.Go(func(stop chan bool) { notify := func(err error, time time.Duration) { log.Errorf("Consul connection error %+v, retrying in %s", err, time) } worker := func() error { return provider.watch(configurationChan, stop) } err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify) if err != nil { log.Fatalf("Cannot connect to consul server %+v", err) } }) return err }
func portUpdate(c *config.Config, ctx context.Context) error { ip, er := getIP(c.OpenVPN.Tun, c.Timeout.Duration, ctx) if er != nil || ctx.Err() != nil { return er } logger.Infof("New bind ip: (%s) %s", c.OpenVPN.Tun, ip) port, er := getPort(ip, c.PIA.User, c.PIA.Pass, c.PIA.ClientID, c.Timeout.Duration, ctx) if er != nil || ctx.Err() != nil { return er } logger.Infof("New peer port: %d", port) notify := func(e error, w time.Duration) { logger.Debugf("Failed to update transmission port: %v", er) } operation := func() error { select { default: return transmission. NewRawClient(c.Transmission.URL.String(), c.Transmission.User, c.Transmission.Pass). UpdatePort(port) case <-ctx.Done(): return nil } } b := backoff.NewExponentialBackOff() b.MaxElapsedTime = c.Timeout.Duration return backoff.RetryNotify(operation, b, notify) }
func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) { var body string var errs []error var resp *http.Response f := func() (err error) { // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() resp, body, errs = gorequest.New().Get(url).End() if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp) } return nil } notify := func(err error, t time.Duration) { log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { errChan <- fmt.Errorf("HTTP Error %s", err) } cveDetail := cve.CveDetail{} if err := json.Unmarshal([]byte(body), &cveDetail); err != nil { errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) } resChan <- response{ key, cveDetail, } }
func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) { operation := func() error { events, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{}) /* stop chan */) if err != nil { log.Errorf("Failed to WatchTree %s", err) return err } for { select { case <-stop: return nil case _, ok := <-events: if !ok { return errors.New("watchtree channel closed") } configuration := provider.loadConfig() if configuration != nil { configurationChan <- types.ConfigMessage{ ProviderName: string(provider.storeType), Configuration: configuration, } } } } } notify := func(err error, time time.Duration) { log.Errorf("KV connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify) if err != nil { log.Fatalf("Cannot connect to KV server %+v", err) } }
func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) { if client = tryAgentConnect(c); client != nil { return client, nil } var auths = []ssh.AuthMethod{} if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil { logrus.Fatalf("Failed to add keyAuth. %s@%s:%s err: %s", c.User, c.Host, c.Port, err) } if c.Password != "" { auths = append(auths, ssh.Password(c.Password)) } // http://blog.ralch.com/tutorial/golang-ssh-connection/ config := &ssh.ClientConfig{ User: c.User, Auth: auths, } notifyFunc := func(e error, t time.Duration) { logrus.Warnf("Failed to ssh %s@%s:%s err: %s, Retrying in %s...", c.User, c.Host, c.Port, e, t) } err = backoff.RetryNotify(func() error { if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil { return err } return nil }, backoff.NewExponentialBackOff(), notifyFunc) return }
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) { var body string var errs []error var resp *http.Response f := func() (err error) { req := gorequest.New().SetDebug(config.Conf.Debug).Post(url) for key := range query { req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json") } resp, body, errs = req.End() if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { return fmt.Errorf("HTTP POST error: %v, url: %s, resp: %v", errs, url, resp) } return nil } notify := func(err error, t time.Duration) { log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err) } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err) } cveDetails := []cve.CveDetail{} if err := json.Unmarshal([]byte(body), &cveDetails); err != nil { return []cve.CveDetail{}, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) } return cveDetails, nil }
func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) { if client = tryAgentConnect(c); client != nil { return client, nil } var auths = []ssh.AuthMethod{} if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil { return nil, err } // http://blog.ralch.com/tutorial/golang-ssh-connection/ config := &ssh.ClientConfig{ User: c.User, Auth: auths, } notifyFunc := func(e error, t time.Duration) { logger := getSSHLogger() logger.Debugf("Failed to Dial to %s, err: %s, Retrying in %s...", c.ServerName, e, t) } err = backoff.RetryNotify(func() error { if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil { return err } return nil }, backoff.NewExponentialBackOff(), notifyFunc) return }
func (c *consulCoordinator) Start(addr net.Addr, errCh chan error) error { if addr == nil { addr = &net.TCPAddr{} } c.addr = addr session := c.client.Session() // set session to delete our keys on invalidation sessionOptions := &api.SessionEntry{ Behavior: api.SessionBehaviorDelete, LockDelay: c.config.LockDelay, TTL: c.config.TTL, } var sessionID string var err error err = backoff.RetryNotify(func() error { sessionID, _, err = session.Create(sessionOptions, nil) return err }, backoff.NewExponentialBackOff(), func(err error, t time.Duration) { log.Println("Cannot create session, retrying in", t, ". Error:", err) }) if err != nil { return fmt.Errorf("failed to create consul session: %v", err) } // set up a long-running goroutine for renewing the session c.sessionRenew = make(chan struct{}) c.sessionID = sessionID log.Println("[coordinator] Coordinator ready") go func() { errCh <- session.RenewPeriodic("5s", sessionID, nil, c.sessionRenew) }() return nil }
// non-blocking func mustWatchServiceDefs(ctx context.Context, client etcd.KeysAPI, basepath *string, changed chan<- bool) { wOpts := &etcd.WatcherOptions{Recursive: true} watcher := client.Watcher(*basepath, wOpts) watchOperation := func() error { resp, err := watcher.Next(ctx) if err != nil { switch v := err.(type) { case etcd.Error: if v.Code == etcd.ErrorCodeEventIndexCleared { watcher = client.Watcher(*basepath, wOpts) log.WithFields(log.Fields{ "basepath": *basepath, "code": v.Code, "cause": v.Cause, "index": v.Index, "message": v.Message, }).Warn("refreshed watcher") return nil } default: if err.Error() == "unexpected end of JSON input" { log.WithField("error", err).Warn("probably a connection timeout. are we in etcd 0.4.x?") return nil } else { return err } } } if resp.Action != "get" { changed <- true } return nil } notify := func(err error, dur time.Duration) { log.WithFields(log.Fields{ "dur": dur, "error": err, "service_path": *basepath, }).Error("service definition watch failed. backing off.") } go func() { for { err := backoff.RetryNotify(watchOperation, backoff.NewExponentialBackOff(), notify) if err != nil { log.WithFields(log.Fields{ "error": err, "service_path": *basepath, }).Fatal("unable to recover communication with etcd, watch abandoned") } } }() }
func main() { cfg, err := New() if err != nil { log.Fatalf("Failed to parse config: %s", err) return } runs := prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "elasticsearch_backup_runs_total", Help: "Number of elasticsearch backup runs", }, []string{"status"}, ) runs = prometheus.MustRegisterOrGet(runs).(*prometheus.CounterVec) duration := prometheus.NewSummaryVec( prometheus.SummaryOpts{ Name: "elasticsearch_backup_duration", Help: "Duration of elasticsearch backup runs", }, []string{"operation"}, ) duration = prometheus.MustRegisterOrGet(duration).(*prometheus.SummaryVec) go listen() interval := time.Hour * time.Duration(cfg.Interval) for { t0 := time.Now() opFunc := func() error { return backupAndRemove(cfg) } logFunc := func(err error, wait time.Duration) { log.Warnf("Failed to connect to ES: %s. Retry in %s", err, wait) } bo := backoff.NewExponentialBackOff() bo.InitialInterval = time.Second bo.MaxInterval = 60 * time.Second bo.MaxElapsedTime = 15 * time.Minute log.Infof("Attempting Snapshot ...") err := backoff.RetryNotify(opFunc, bo, logFunc) if err != nil { runs.WithLabelValues("failed").Inc() log.Warnf("Failed to delete snapshots: %s", err) continue } runs.WithLabelValues("ok").Inc() d0 := float64(time.Since(t0)) / float64(time.Microsecond) duration.WithLabelValues("backup").Observe(d0) if interval < time.Second { break } log.Infof("Waiting %s until next run", interval.String()) time.Sleep(interval) } os.Exit(0) }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage) error { var dockerClient *docker.Client var err error if provider.TLS != nil { dockerClient, err = docker.NewTLSClient(provider.Endpoint, provider.TLS.Cert, provider.TLS.Key, provider.TLS.CA) } else { dockerClient, err = docker.NewClient(provider.Endpoint) } if err != nil { log.Errorf("Failed to create a client for docker, error: %s", err) return err } err = dockerClient.Ping() if err != nil { log.Errorf("Docker connection error %+v", err) return err } log.Debug("Docker connection established") if provider.Watch { dockerEvents := make(chan *docker.APIEvents) dockerClient.AddEventListener(dockerEvents) log.Debug("Docker listening") go func() { operation := func() error { for { event := <-dockerEvents if event == nil { return errors.New("Docker event nil") // log.Fatalf("Docker connection error") } if event.Status == "start" || event.Status == "die" { log.Debugf("Docker event receveived %+v", event) configuration := provider.loadDockerConfig(dockerClient) if configuration != nil { configurationChan <- types.ConfigMessage{"docker", configuration} } } } } notify := func(err error, time time.Duration) { log.Errorf("Docker connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify) if err != nil { log.Fatalf("Cannot connect to docker server %+v", err) } }() } configuration := provider.loadDockerConfig(dockerClient) configurationChan <- types.ConfigMessage{"docker", configuration} return nil }
// Retry is the core library method for retrying http calls. // // httpCall should be a function that performs the http operation, and returns // (resp *http.Response, tempError error, permError error). Errors that should // cause retries should be returned as tempError. Permanent errors that should // not result in retries should be returned as permError. Retries are performed // using the exponential backoff algorithm from the github.com/cenkalti/backoff // package. Retry automatically treats HTTP 5xx status codes as a temporary // error, and any other non-2xx HTTP status codes as a permanent error. Thus // httpCall function does not need to handle the HTTP status code of resp, // since Retry will take care of it. // // Concurrent use of this library method is supported. func (httpRetryClient *Client) Retry(httpCall func() (resp *http.Response, tempError error, permError error)) (*http.Response, int, error) { var tempError, permError error var response *http.Response attempts := 0 doHttpCall := func() error { response, tempError, permError = httpCall() attempts += 1 if tempError != nil { return tempError } if permError != nil { return nil } // only call this if there is a non 2xx response body := func(response *http.Response) string { // this is a no-op raw, err := httputil.DumpResponse(response, true) if err == nil { return string(raw) } return "" } // now check if http response code is such that we should retry [500, 600)... if respCode := response.StatusCode; respCode/100 == 5 { return BadHttpResponseCode{ HttpResponseCode: respCode, Message: "(Intermittent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response), } } // now check http response code is ok [200, 300)... if respCode := response.StatusCode; respCode/100 != 2 { permError = BadHttpResponseCode{ HttpResponseCode: respCode, Message: "(Permanent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response), } return nil } return nil } // Make HTTP API calls using an exponential backoff algorithm... b := backoff.ExponentialBackOff(*httpRetryClient.BackOffSettings) backoff.RetryNotify(doHttpCall, &b, func(err error, wait time.Duration) { log.Printf("Error: %s", err) }) switch { case permError != nil: return response, attempts, permError case tempError != nil: return response, attempts, tempError default: return response, attempts, nil } }
func withRetries(f func() error) error { backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = time.Second backoffConfig.MaxInterval = 10 * time.Second backoffConfig.MaxElapsedTime = 60 * time.Second notifyFunc := func(err error, dur time.Duration) { log.Printf("waiting %v, failed to get move from player: %s", dur, err) } err := backoff.RetryNotify(f, backoffConfig, notifyFunc) return err }
func (c *Client) CleanTorrents() error { logger.Infof("Running torrent cleaner") torrents, er := c.GetTorrents() if er != nil { return er } torrents.SortByID(false) logger.Infof("Found %d torrents to process", len(torrents)) for _, t := range torrents { logger.Debugf("[Torrent %d: %q] Checking status", t.ID, t.Name) id := util.Hashf(md5.New(), t.ID, t.Name) status := &torrentStatus{Torrent: t, id: id, failures: 0} status.setFailures() if st, ok := seen[id]; ok { status.failures = status.failures + st.failures if !updated(st.Torrent, status.Torrent) { status.failures++ } } seen[id] = status logger.Debugf("[Torrent %d: %q] Failures: %d", t.ID, t.Name, status.failures) } b := backoff.NewExponentialBackOff() b.MaxElapsedTime = 15 * time.Second remove := make([]*torrentStatus, 0, 1) for _, t := range seen { if t.failed() { b.Reset() logger.Infof("[Torrent %d: %q] Removing", t.ID, t.Name) er := backoff.RetryNotify(delTorrent(c, t.Torrent), b, func(e error, w time.Duration) { logger.Errorf("[Torrent %d: %q] Failed to remove (retry in %v): %v", t.ID, t.Name, w, e) }) if er == nil { remove = append(remove, t) } else { logger.Errorf("[Torrent %d: %q] Failed to remove, will retry next cycle", t.ID, t.Name) } } } for i := range remove { delete(seen, remove[i].id) } return nil }
// RoundTrip implements the RoundTripper interface. func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) { // Initialize the exponential backoff client. backOffClient := &backoff.ExponentialBackOff{ InitialInterval: t.backOffConfig.initialInterval, RandomizationFactor: t.backOffConfig.randomizationFactor, Multiplier: t.backOffConfig.backOffMultiplier, MaxInterval: t.backOffConfig.maxInterval, MaxElapsedTime: t.backOffConfig.maxElapsedTime, Clock: backoff.SystemClock, } // Make a copy of the request's Body so that we can reuse it if the request // needs to be backed off and retried. bodyBuf := bytes.Buffer{} if req.Body != nil { if _, err := bodyBuf.ReadFrom(req.Body); err != nil { return nil, fmt.Errorf("Failed to read request body: %v", err) } } var resp *http.Response var err error roundTripOp := func() error { if req.Body != nil { req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String())) } resp, err = t.Transport.RoundTrip(req) if err != nil { return fmt.Errorf("Error while making the round trip: %s", err) } if resp != nil { if resp.StatusCode >= 500 && resp.StatusCode <= 599 { return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL) } else if resp.StatusCode < 200 || resp.StatusCode > 299 { // Stop backing off if there are non server errors. backOffClient.MaxElapsedTime = backoff.Stop return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL) } } return nil } notifyFunc := func(err error, wait time.Duration) { glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait) } if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil { return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err) } return resp, nil }
func restartProcesses(t, v *process.Process, c *config.Config, ctx context.Context) error { var ( notify = func(e error, t time.Duration) { logger.Errorf("Failed to restart processes (retry in %v): %v", t, e) } operation = func() error { t.Stop() v.Stop() return startProcesses(t, v, c, ctx) } b = backoff.NewExponentialBackOff() ) b.MaxElapsedTime = 30 * time.Minute b.MaxInterval = 10 * time.Second return backoff.RetryNotify(operation, b, notify) }
func getIP(dev string, timeout time.Duration, c context.Context) (string, error) { var address string notify := func(e error, w time.Duration) { logger.Errorf("Failed to get IP for %q (retry in %v): %v", dev, w, e) } fn := func() (er error) { select { default: address, er = vpn.FindIP(dev) return case <-c.Done(): return } } b := backoff.NewExponentialBackOff() b.MaxElapsedTime = timeout return address, backoff.RetryNotify(fn, b, notify) }
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region. // It will then return a channel that will eventually be closed when leadership is rescinded. func RegionLeader(id string) Leader { path := fmt.Sprintf(regionLeaderPath, id) prefix := path + "/lock-" var lockNode string for { // create our lock node -- retry until this is done, use exponential backoff // to add some delay between attempts b := backoff.NewExponentialBackOff() b.InitialInterval = backoffInitialInterval b.MaxInterval = backoffMaxInterval b.MaxElapsedTime = 0 // Never stop retrying backoff.RetryNotify(func() (err error) { log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election") lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll)) return }, b, func(err error, d time.Duration) { if err == gozk.ErrNoNode { createParents(path) } else if err != nil { log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d) } }) err := waitForWinner(path, lockNode) if err != nil { // try to cleanup - then go again zookeeper.Delete(lockNode, -1) time.Sleep(time.Second) continue } // we are the leader break } log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id) inst.Counter(1.0, "sync.regionleader.elected") return newRegionLeader(lockNode) }
// DoCommand executes the specific rpc command wrapped in rpcFunc. func (c *Connection) DoCommand(ctx context.Context, rpcFunc func(keybase1.GenericClient) error) error { for { // we may or may not be in the process of reconnecting. // if so we'll block here unless canceled by the caller. connErr := c.waitForConnection(ctx) if connErr != nil { return connErr } var rpcErr error // retry throttle errors w/backoff throttleErr := backoff.RetryNotify(func() error { rawClient := func() keybase1.GenericClient { c.mutex.Lock() defer c.mutex.Unlock() return c.client }() // try the rpc call. this can also be canceled // by the caller, and will retry connectivity // errors w/backoff. throttleErr := runUnlessCanceled(ctx, func() error { return rpcFunc(rawClient) }) if c.handler.ShouldThrottle(throttleErr) { return throttleErr } rpcErr = throttleErr return nil }, backoff.NewExponentialBackOff(), c.handler.OnDoCommandError) // RetryNotify gave up. if throttleErr != nil { return throttleErr } // check to see if we need to retry it. if !c.checkForRetry(rpcErr) { return rpcErr } } }
func (b *BackoffReadCloser) Read(data []byte) (int, error) { bytesRead := 0 var n int var err error backoff.RetryNotify(func() error { n, err = b.reader.Read(data[bytesRead:]) bytesRead += n if b.client.IsRetryable(err) { return err } return nil }, b.backoffConfig, func(err error, d time.Duration) { protolion.Infof("Error reading (retrying): %#v", RetryError{ err: err.Error(), timeTillNextRetry: d.String(), bytesProcessed: bytesRead, }) }) return bytesRead, err }
// discover attempts to find new nodes in the cluster using the current nodes func (c *Cluster) discover() { // Keep retrying with exponential backoff. b := backoff.NewExponentialBackOff() // Never finish retrying (max interval is still 60s) b.MaxElapsedTime = 0 // Keep trying to discover new nodes for { backoff.RetryNotify(func() error { // If no hosts try seeding nodes if len(c.GetNodes()) == 0 { c.connectNodes(c.getSeeds()) } return c.listenForNodeChanges() }, b, func(err error, wait time.Duration) { Log.Debugf("Error discovering hosts %s, waiting: %s", err, wait) }) } }
// DoCommand executes the specific rpc command wrapped in rpcFunc. func (c *Connection) DoCommand(ctx context.Context, name string, rpcFunc func(GenericClient) error) error { for { // we may or may not be in the process of reconnecting. // if so we'll block here unless canceled by the caller. connErr := c.waitForConnection(ctx) if connErr != nil { return connErr } var rpcErr error // retry throttle errors w/backoff throttleErr := backoff.RetryNotify(func() error { rawClient := func() GenericClient { c.mutex.Lock() defer c.mutex.Unlock() return c.client }() // try the rpc call, assuming that it exits // immediately when ctx is canceled. will // retry connectivity errors w/backoff. throttleErr := rpcFunc(rawClient) if throttleErr != nil && c.handler.ShouldRetry(name, throttleErr) { return throttleErr } rpcErr = throttleErr return nil }, c.doCommandBackoff, c.handler.OnDoCommandError) // RetryNotify gave up. if throttleErr != nil { return throttleErr } // check to see if we need to retry it. if !c.checkForRetry(rpcErr) { return rpcErr } } }
// DoRequestWithExponentialBackOff makes an http request using the http.Client and http.Request associated with this context. // You can pass a condition and a BackOff configuration. See https://github.com/cenkalti/backoff to know more about backoff. // If no BackOff is provided it will use the default exponential BackOff configuration. // See also ErrorIfStatusCodeIsNot function that provides a basic condition based on status code. func (c *Context) DoRequestWithExponentialBackOff(condition BackoffCondition, b backoff.BackOff) (*http.Response, error) { if b == nil { b = backoff.NewExponentialBackOff() } err := backoff.RetryNotify( func() error { var err error res, err := c.DoRequest() if err != nil { return err } return condition(res) }, b, func(err error, wait time.Duration) { fmt.Println("˙\nBackoff:Waiting: ", wait) fmt.Println(err) fmt.Println("") }) return c.Response(), err }
func (w SlackWriter) Write(scanResults []models.ScanResult) error { conf := config.Conf.Slack for _, s := range scanResults { channel := conf.Channel if channel == "${servername}" { channel = fmt.Sprintf("#%s", s.ServerName) } msg := message{ Text: msgText(s), Username: conf.AuthUser, IconEmoji: conf.IconEmoji, Channel: channel, Attachments: toSlackAttachments(s), } bytes, _ := json.Marshal(msg) jsonBody := string(bytes) f := func() (err error) { resp, body, errs := gorequest.New().Proxy(config.Conf.HTTPProxy).Post(conf.HookURL). Send(string(jsonBody)).End() if resp.StatusCode != 200 { log.Errorf("Resonse body: %s", body) if len(errs) > 0 { return errs[0] } } return nil } notify := func(err error, t time.Duration) { log.Warn("Retrying in ", t) } if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil { return fmt.Errorf("HTTP Error: %s", err) } } return nil }
// doReconnect attempts a reconnection. It assumes that reconnectChan // and reconnectErrPtr are the same ones in c, but are passed in to // avoid having to take the mutex at the beginning of the method. func (c *Connection) doReconnect(ctx context.Context, disconnectStatus DisconnectStatus, reconnectChan chan struct{}, reconnectErrPtr *error) { // inform the handler of our disconnected state c.handler.OnDisconnected(ctx, disconnectStatus) err := backoff.RetryNotify(func() error { // try to connect err := c.connect(ctx) select { case <-ctx.Done(): // context was canceled by Shutdown() or a user action *reconnectErrPtr = ctx.Err() // short-circuit Retry return nil default: } if !c.handler.ShouldRetryOnConnect(err) { // A fatal error happened. *reconnectErrPtr = err // short-circuit Retry return nil } return err }, c.reconnectBackoff, // give the caller a chance to log any other error or adjust state c.handler.OnConnectError) if err != nil { // this shouldn't happen, but just in case. *reconnectErrPtr = err } // close the reconnect channel to signal we're connected. c.mutex.Lock() defer c.mutex.Unlock() close(reconnectChan) c.reconnectChan = nil c.cancelFunc = nil c.reconnectErrPtr = nil }
func getPort(ip, user, pass, id string, timeout time.Duration, c context.Context) (int, error) { var port int notify := func(e error, w time.Duration) { logger.Errorf("Failed to get port from PIA (retry in %v): %v", w, e) } fn := func() error { select { default: p, er := pia.RequestPort(ip, user, pass, id) if er != nil { return er } port = p return nil case <-c.Done(): return nil } } b := backoff.NewExponentialBackOff() b.MaxElapsedTime = timeout return port, backoff.RetryNotify(fn, b, notify) }
// Provide allows the provider to provide configurations to traefik // using the given configuration channel. func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage) error { config := api.DefaultConfig() config.Address = provider.Endpoint client, err := api.NewClient(config) if err != nil { return err } provider.client = client go func() { notify := func(err error, time time.Duration) { log.Errorf("Consul connection error %+v, retrying in %s", err, time) } worker := func() error { return provider.watch(configurationChan) } err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify) if err != nil { log.Fatalf("Cannot connect to consul server %+v", err) } }() return err }
func (a *apiServer) AddShard(shard uint64) error { persistClient, err := a.getPersistClient() if err != nil { return err } ctx, cancel := context.WithCancel(context.Background()) a.shardCancelFuncsLock.Lock() defer a.shardCancelFuncsLock.Unlock() if _, ok := a.shardCancelFuncs[shard]; ok { return fmt.Errorf("shard %d is being added twice; this is likely a bug", shard) } a.shardCancelFuncs[shard] = cancel client, err := persistClient.SubscribePipelineInfos(ctx, &persist.SubscribePipelineInfosRequest{ IncludeInitial: true, Shard: &persist.Shard{shard}, }) if err != nil { return err } go func() { for { pipelineChange, err := client.Recv() if err != nil { return } if pipelineChange.Removed { a.cancelFuncsLock.Lock() cancel, ok := a.cancelFuncs[pipelineChange.Pipeline.PipelineName] if ok { cancel() delete(a.cancelFuncs, pipelineChange.Pipeline.PipelineName) } else { protolion.Printf("trying to cancel a pipeline that we are not assigned to; this is likely a bug") } a.cancelFuncsLock.Unlock() } else { // We only want to start a goro to run the pipeline if the // pipeline has more than one inputs go func() { b := backoff.NewExponentialBackOff() // We set MaxElapsedTime to 0 because we want the retry to // never stop. // However, ideally we should crash this pps server so the // pipeline gets reassigned to another pps server. // The reason we don't do that right now is that pps and // pfs are bundled together, so by crashing this program // we will be crashing a pfs node too, which might cause // cascading failures as other pps nodes might be depending // on it. b.MaxElapsedTime = 0 err = backoff.RetryNotify(func() error { if err := a.runPipeline(newPipelineInfo(pipelineChange.Pipeline)); err != nil && !isContextCancelled(err) { return err } return nil }, b, func(err error, d time.Duration) { if _, err = persistClient.UpdatePipelineState(context.Background(), &persist.UpdatePipelineStateRequest{ PipelineName: pipelineChange.Pipeline.PipelineName, State: ppsclient.PipelineState_PIPELINE_RESTARTING, RecentError: err.Error(), }); err != nil { protolion.Errorf("error updating pipeline state: %v", err) } }) // At this point we stop retrying and update the pipeline state // to FAILED if err != nil { if _, err = persistClient.UpdatePipelineState(context.Background(), &persist.UpdatePipelineStateRequest{ PipelineName: pipelineChange.Pipeline.PipelineName, State: ppsclient.PipelineState_PIPELINE_FAILURE, RecentError: err.Error(), }); err != nil { protolion.Errorf("error updating pipeline state: %v", err) } } }() } } }() return nil }
func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool) error { storeConfig := &store.Config{ ConnectionTimeout: 30 * time.Second, Bucket: "traefik", } if provider.TLS != nil { caPool := x509.NewCertPool() if provider.TLS.CA != "" { ca, err := ioutil.ReadFile(provider.TLS.CA) if err != nil { return fmt.Errorf("Failed to read CA. %s", err) } caPool.AppendCertsFromPEM(ca) } cert, err := tls.LoadX509KeyPair(provider.TLS.Cert, provider.TLS.Key) if err != nil { return fmt.Errorf("Failed to load keypair. %s", err) } storeConfig.TLS = &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caPool, InsecureSkipVerify: provider.TLS.InsecureSkipVerify, } } operation := func() error { kv, err := libkv.NewStore( provider.storeType, strings.Split(provider.Endpoint, ","), storeConfig, ) if err != nil { return err } if _, err := kv.List(""); err != nil { return err } provider.kvclient = kv if provider.Watch { pool.Go(func(stop chan bool) { provider.watchKv(configurationChan, provider.Prefix, stop) }) } configuration := provider.loadConfig() configurationChan <- types.ConfigMessage{ ProviderName: string(provider.storeType), Configuration: configuration, } return nil } notify := func(err error, time time.Duration) { log.Errorf("KV connection error %+v, retrying in %s", err, time) } err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify) if err != nil { log.Fatalf("Cannot connect to KV server %+v", err) } return nil }