// doReconnect attempts a reconnection. It assumes that reconnectChan
// and reconnectErrPtr are the same ones in c, but are passed in to
// avoid having to take the mutex at the beginning of the method.
func (c *Connection) doReconnect(ctx context.Context,
reconnectChan chan struct{}, reconnectErrPtr *error) {
// retry w/exponential backoff
backoff.RetryNotify(func() error {
// try to connect
err := c.connect(ctx)
select {
case <-ctx.Done():
// context was canceled by Shutdown() or a user action
*reconnectErrPtr = ctx.Err()
// short-circuit Retry
return nil
default:
}
if dontRetryOnConnect(err) {
// A fatal error happened.
*reconnectErrPtr = err
// short-circuit Retry
return nil
}
return err
}, backoff.NewExponentialBackOff(),
// give the caller a chance to log any other error or adjust state
c.handler.OnConnectError)
// close the reconnect channel to signal we're connected.
c.mutex.Lock()
defer c.mutex.Unlock()
close(reconnectChan)
c.reconnectChan = nil
c.cancelFunc = nil
c.reconnectErrPtr = nil
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints []types.Constraint) error {
config := api.DefaultConfig()
config.Address = provider.Endpoint
client, err := api.NewClient(config)
if err != nil {
return err
}
provider.client = client
provider.Constraints = append(provider.Constraints, constraints...)
pool.Go(func(stop chan bool) {
notify := func(err error, time time.Duration) {
log.Errorf("Consul connection error %+v, retrying in %s", err, time)
}
worker := func() error {
return provider.watch(configurationChan, stop)
}
err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to consul server %+v", err)
}
})
return err
}
func portUpdate(c *config.Config, ctx context.Context) error {
ip, er := getIP(c.OpenVPN.Tun, c.Timeout.Duration, ctx)
if er != nil || ctx.Err() != nil {
return er
}
logger.Infof("New bind ip: (%s) %s", c.OpenVPN.Tun, ip)
port, er := getPort(ip, c.PIA.User, c.PIA.Pass, c.PIA.ClientID, c.Timeout.Duration, ctx)
if er != nil || ctx.Err() != nil {
return er
}
logger.Infof("New peer port: %d", port)
notify := func(e error, w time.Duration) {
logger.Debugf("Failed to update transmission port: %v", er)
}
operation := func() error {
select {
default:
return transmission.
NewRawClient(c.Transmission.URL.String(), c.Transmission.User, c.Transmission.Pass).
UpdatePort(port)
case <-ctx.Done():
return nil
}
}
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = c.Timeout.Duration
return backoff.RetryNotify(operation, b, notify)
}
func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp)
}
return nil
}
notify := func(err error, t time.Duration) {
log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
errChan <- fmt.Errorf("HTTP Error %s", err)
}
cveDetail := cve.CveDetail{}
if err := json.Unmarshal([]byte(body), &cveDetail); err != nil {
errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
}
resChan <- response{
key,
cveDetail,
}
}
func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) {
operation := func() error {
events, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{}) /* stop chan */)
if err != nil {
log.Errorf("Failed to WatchTree %s", err)
return err
}
for {
select {
case <-stop:
return nil
case _, ok := <-events:
if !ok {
return errors.New("watchtree channel closed")
}
configuration := provider.loadConfig()
if configuration != nil {
configurationChan <- types.ConfigMessage{
ProviderName: string(provider.storeType),
Configuration: configuration,
}
}
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to KV server %+v", err)
}
}
func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) {
if client = tryAgentConnect(c); client != nil {
return client, nil
}
var auths = []ssh.AuthMethod{}
if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil {
logrus.Fatalf("Failed to add keyAuth. %s@%s:%s err: %s",
c.User, c.Host, c.Port, err)
}
if c.Password != "" {
auths = append(auths, ssh.Password(c.Password))
}
// http://blog.ralch.com/tutorial/golang-ssh-connection/
config := &ssh.ClientConfig{
User: c.User,
Auth: auths,
}
notifyFunc := func(e error, t time.Duration) {
logrus.Warnf("Failed to ssh %s@%s:%s err: %s, Retrying in %s...",
c.User, c.Host, c.Port, e, t)
}
err = backoff.RetryNotify(func() error {
if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil {
return err
}
return nil
}, backoff.NewExponentialBackOff(), notifyFunc)
return
}
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) {
var body string
var errs []error
var resp *http.Response
f := func() (err error) {
req := gorequest.New().SetDebug(config.Conf.Debug).Post(url)
for key := range query {
req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
}
resp, body, errs = req.End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return fmt.Errorf("HTTP POST error: %v, url: %s, resp: %v", errs, url, resp)
}
return nil
}
notify := func(err error, t time.Duration) {
log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
}
cveDetails := []cve.CveDetail{}
if err := json.Unmarshal([]byte(body), &cveDetails); err != nil {
return []cve.CveDetail{},
fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
}
return cveDetails, nil
}
func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) {
if client = tryAgentConnect(c); client != nil {
return client, nil
}
var auths = []ssh.AuthMethod{}
if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil {
return nil, err
}
// http://blog.ralch.com/tutorial/golang-ssh-connection/
config := &ssh.ClientConfig{
User: c.User,
Auth: auths,
}
notifyFunc := func(e error, t time.Duration) {
logger := getSSHLogger()
logger.Debugf("Failed to Dial to %s, err: %s, Retrying in %s...",
c.ServerName, e, t)
}
err = backoff.RetryNotify(func() error {
if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil {
return err
}
return nil
}, backoff.NewExponentialBackOff(), notifyFunc)
return
}
func (c *consulCoordinator) Start(addr net.Addr, errCh chan error) error {
if addr == nil {
addr = &net.TCPAddr{}
}
c.addr = addr
session := c.client.Session()
// set session to delete our keys on invalidation
sessionOptions := &api.SessionEntry{
Behavior: api.SessionBehaviorDelete,
LockDelay: c.config.LockDelay,
TTL: c.config.TTL,
}
var sessionID string
var err error
err = backoff.RetryNotify(func() error {
sessionID, _, err = session.Create(sessionOptions, nil)
return err
}, backoff.NewExponentialBackOff(), func(err error, t time.Duration) {
log.Println("Cannot create session, retrying in", t, ". Error:", err)
})
if err != nil {
return fmt.Errorf("failed to create consul session: %v", err)
}
// set up a long-running goroutine for renewing the session
c.sessionRenew = make(chan struct{})
c.sessionID = sessionID
log.Println("[coordinator] Coordinator ready")
go func() {
errCh <- session.RenewPeriodic("5s", sessionID, nil, c.sessionRenew)
}()
return nil
}
// non-blocking
func mustWatchServiceDefs(ctx context.Context, client etcd.KeysAPI, basepath *string, changed chan<- bool) {
wOpts := &etcd.WatcherOptions{Recursive: true}
watcher := client.Watcher(*basepath, wOpts)
watchOperation := func() error {
resp, err := watcher.Next(ctx)
if err != nil {
switch v := err.(type) {
case etcd.Error:
if v.Code == etcd.ErrorCodeEventIndexCleared {
watcher = client.Watcher(*basepath, wOpts)
log.WithFields(log.Fields{
"basepath": *basepath,
"code": v.Code,
"cause": v.Cause,
"index": v.Index,
"message": v.Message,
}).Warn("refreshed watcher")
return nil
}
default:
if err.Error() == "unexpected end of JSON input" {
log.WithField("error", err).Warn("probably a connection timeout. are we in etcd 0.4.x?")
return nil
} else {
return err
}
}
}
if resp.Action != "get" {
changed <- true
}
return nil
}
notify := func(err error, dur time.Duration) {
log.WithFields(log.Fields{
"dur": dur,
"error": err,
"service_path": *basepath,
}).Error("service definition watch failed. backing off.")
}
go func() {
for {
err := backoff.RetryNotify(watchOperation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"service_path": *basepath,
}).Fatal("unable to recover communication with etcd, watch abandoned")
}
}
}()
}
func main() {
cfg, err := New()
if err != nil {
log.Fatalf("Failed to parse config: %s", err)
return
}
runs := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "elasticsearch_backup_runs_total",
Help: "Number of elasticsearch backup runs",
},
[]string{"status"},
)
runs = prometheus.MustRegisterOrGet(runs).(*prometheus.CounterVec)
duration := prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "elasticsearch_backup_duration",
Help: "Duration of elasticsearch backup runs",
},
[]string{"operation"},
)
duration = prometheus.MustRegisterOrGet(duration).(*prometheus.SummaryVec)
go listen()
interval := time.Hour * time.Duration(cfg.Interval)
for {
t0 := time.Now()
opFunc := func() error {
return backupAndRemove(cfg)
}
logFunc := func(err error, wait time.Duration) {
log.Warnf("Failed to connect to ES: %s. Retry in %s", err, wait)
}
bo := backoff.NewExponentialBackOff()
bo.InitialInterval = time.Second
bo.MaxInterval = 60 * time.Second
bo.MaxElapsedTime = 15 * time.Minute
log.Infof("Attempting Snapshot ...")
err := backoff.RetryNotify(opFunc, bo, logFunc)
if err != nil {
runs.WithLabelValues("failed").Inc()
log.Warnf("Failed to delete snapshots: %s", err)
continue
}
runs.WithLabelValues("ok").Inc()
d0 := float64(time.Since(t0)) / float64(time.Microsecond)
duration.WithLabelValues("backup").Observe(d0)
if interval < time.Second {
break
}
log.Infof("Waiting %s until next run", interval.String())
time.Sleep(interval)
}
os.Exit(0)
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage) error {
var dockerClient *docker.Client
var err error
if provider.TLS != nil {
dockerClient, err = docker.NewTLSClient(provider.Endpoint,
provider.TLS.Cert, provider.TLS.Key, provider.TLS.CA)
} else {
dockerClient, err = docker.NewClient(provider.Endpoint)
}
if err != nil {
log.Errorf("Failed to create a client for docker, error: %s", err)
return err
}
err = dockerClient.Ping()
if err != nil {
log.Errorf("Docker connection error %+v", err)
return err
}
log.Debug("Docker connection established")
if provider.Watch {
dockerEvents := make(chan *docker.APIEvents)
dockerClient.AddEventListener(dockerEvents)
log.Debug("Docker listening")
go func() {
operation := func() error {
for {
event := <-dockerEvents
if event == nil {
return errors.New("Docker event nil")
// log.Fatalf("Docker connection error")
}
if event.Status == "start" || event.Status == "die" {
log.Debugf("Docker event receveived %+v", event)
configuration := provider.loadDockerConfig(dockerClient)
if configuration != nil {
configurationChan <- types.ConfigMessage{"docker", configuration}
}
}
}
}
notify := func(err error, time time.Duration) {
log.Errorf("Docker connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to docker server %+v", err)
}
}()
}
configuration := provider.loadDockerConfig(dockerClient)
configurationChan <- types.ConfigMessage{"docker", configuration}
return nil
}
// Retry is the core library method for retrying http calls.
//
// httpCall should be a function that performs the http operation, and returns
// (resp *http.Response, tempError error, permError error). Errors that should
// cause retries should be returned as tempError. Permanent errors that should
// not result in retries should be returned as permError. Retries are performed
// using the exponential backoff algorithm from the github.com/cenkalti/backoff
// package. Retry automatically treats HTTP 5xx status codes as a temporary
// error, and any other non-2xx HTTP status codes as a permanent error. Thus
// httpCall function does not need to handle the HTTP status code of resp,
// since Retry will take care of it.
//
// Concurrent use of this library method is supported.
func (httpRetryClient *Client) Retry(httpCall func() (resp *http.Response, tempError error, permError error)) (*http.Response, int, error) {
var tempError, permError error
var response *http.Response
attempts := 0
doHttpCall := func() error {
response, tempError, permError = httpCall()
attempts += 1
if tempError != nil {
return tempError
}
if permError != nil {
return nil
}
// only call this if there is a non 2xx response
body := func(response *http.Response) string {
// this is a no-op
raw, err := httputil.DumpResponse(response, true)
if err == nil {
return string(raw)
}
return ""
}
// now check if http response code is such that we should retry [500, 600)...
if respCode := response.StatusCode; respCode/100 == 5 {
return BadHttpResponseCode{
HttpResponseCode: respCode,
Message: "(Intermittent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response),
}
}
// now check http response code is ok [200, 300)...
if respCode := response.StatusCode; respCode/100 != 2 {
permError = BadHttpResponseCode{
HttpResponseCode: respCode,
Message: "(Permanent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response),
}
return nil
}
return nil
}
// Make HTTP API calls using an exponential backoff algorithm...
b := backoff.ExponentialBackOff(*httpRetryClient.BackOffSettings)
backoff.RetryNotify(doHttpCall, &b, func(err error, wait time.Duration) {
log.Printf("Error: %s", err)
})
switch {
case permError != nil:
return response, attempts, permError
case tempError != nil:
return response, attempts, tempError
default:
return response, attempts, nil
}
}
func withRetries(f func() error) error {
backoffConfig := backoff.NewExponentialBackOff()
backoffConfig.InitialInterval = time.Second
backoffConfig.MaxInterval = 10 * time.Second
backoffConfig.MaxElapsedTime = 60 * time.Second
notifyFunc := func(err error, dur time.Duration) {
log.Printf("waiting %v, failed to get move from player: %s", dur, err)
}
err := backoff.RetryNotify(f, backoffConfig, notifyFunc)
return err
}
func (c *Client) CleanTorrents() error {
logger.Infof("Running torrent cleaner")
torrents, er := c.GetTorrents()
if er != nil {
return er
}
torrents.SortByID(false)
logger.Infof("Found %d torrents to process", len(torrents))
for _, t := range torrents {
logger.Debugf("[Torrent %d: %q] Checking status", t.ID, t.Name)
id := util.Hashf(md5.New(), t.ID, t.Name)
status := &torrentStatus{Torrent: t, id: id, failures: 0}
status.setFailures()
if st, ok := seen[id]; ok {
status.failures = status.failures + st.failures
if !updated(st.Torrent, status.Torrent) {
status.failures++
}
}
seen[id] = status
logger.Debugf("[Torrent %d: %q] Failures: %d", t.ID, t.Name, status.failures)
}
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = 15 * time.Second
remove := make([]*torrentStatus, 0, 1)
for _, t := range seen {
if t.failed() {
b.Reset()
logger.Infof("[Torrent %d: %q] Removing", t.ID, t.Name)
er := backoff.RetryNotify(delTorrent(c, t.Torrent), b, func(e error, w time.Duration) {
logger.Errorf("[Torrent %d: %q] Failed to remove (retry in %v): %v", t.ID, t.Name, w, e)
})
if er == nil {
remove = append(remove, t)
} else {
logger.Errorf("[Torrent %d: %q] Failed to remove, will retry next cycle", t.ID, t.Name)
}
}
}
for i := range remove {
delete(seen, remove[i].id)
}
return nil
}
// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Initialize the exponential backoff client.
backOffClient := &backoff.ExponentialBackOff{
InitialInterval: t.backOffConfig.initialInterval,
RandomizationFactor: t.backOffConfig.randomizationFactor,
Multiplier: t.backOffConfig.backOffMultiplier,
MaxInterval: t.backOffConfig.maxInterval,
MaxElapsedTime: t.backOffConfig.maxElapsedTime,
Clock: backoff.SystemClock,
}
// Make a copy of the request's Body so that we can reuse it if the request
// needs to be backed off and retried.
bodyBuf := bytes.Buffer{}
if req.Body != nil {
if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
return nil, fmt.Errorf("Failed to read request body: %v", err)
}
}
var resp *http.Response
var err error
roundTripOp := func() error {
if req.Body != nil {
req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
}
resp, err = t.Transport.RoundTrip(req)
if err != nil {
return fmt.Errorf("Error while making the round trip: %s", err)
}
if resp != nil {
if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
// Stop backing off if there are non server errors.
backOffClient.MaxElapsedTime = backoff.Stop
return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
}
}
return nil
}
notifyFunc := func(err error, wait time.Duration) {
glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
}
if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
}
return resp, nil
}
func restartProcesses(t, v *process.Process, c *config.Config, ctx context.Context) error {
var (
notify = func(e error, t time.Duration) {
logger.Errorf("Failed to restart processes (retry in %v): %v", t, e)
}
operation = func() error {
t.Stop()
v.Stop()
return startProcesses(t, v, c, ctx)
}
b = backoff.NewExponentialBackOff()
)
b.MaxElapsedTime = 30 * time.Minute
b.MaxInterval = 10 * time.Second
return backoff.RetryNotify(operation, b, notify)
}
func getIP(dev string, timeout time.Duration, c context.Context) (string, error) {
var address string
notify := func(e error, w time.Duration) {
logger.Errorf("Failed to get IP for %q (retry in %v): %v", dev, w, e)
}
fn := func() (er error) {
select {
default:
address, er = vpn.FindIP(dev)
return
case <-c.Done():
return
}
}
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = timeout
return address, backoff.RetryNotify(fn, b, notify)
}
// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region.
// It will then return a channel that will eventually be closed when leadership is rescinded.
func RegionLeader(id string) Leader {
path := fmt.Sprintf(regionLeaderPath, id)
prefix := path + "/lock-"
var lockNode string
for {
// create our lock node -- retry until this is done, use exponential backoff
// to add some delay between attempts
b := backoff.NewExponentialBackOff()
b.InitialInterval = backoffInitialInterval
b.MaxInterval = backoffMaxInterval
b.MaxElapsedTime = 0 // Never stop retrying
backoff.RetryNotify(func() (err error) {
log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election")
lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll))
return
}, b, func(err error, d time.Duration) {
if err == gozk.ErrNoNode {
createParents(path)
} else if err != nil {
log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d)
}
})
err := waitForWinner(path, lockNode)
if err != nil {
// try to cleanup - then go again
zookeeper.Delete(lockNode, -1)
time.Sleep(time.Second)
continue
}
// we are the leader
break
}
log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id)
inst.Counter(1.0, "sync.regionleader.elected")
return newRegionLeader(lockNode)
}
// DoCommand executes the specific rpc command wrapped in rpcFunc.
func (c *Connection) DoCommand(ctx context.Context, rpcFunc func(keybase1.GenericClient) error) error {
for {
// we may or may not be in the process of reconnecting.
// if so we'll block here unless canceled by the caller.
connErr := c.waitForConnection(ctx)
if connErr != nil {
return connErr
}
var rpcErr error
// retry throttle errors w/backoff
throttleErr := backoff.RetryNotify(func() error {
rawClient := func() keybase1.GenericClient {
c.mutex.Lock()
defer c.mutex.Unlock()
return c.client
}()
// try the rpc call. this can also be canceled
// by the caller, and will retry connectivity
// errors w/backoff.
throttleErr := runUnlessCanceled(ctx, func() error {
return rpcFunc(rawClient)
})
if c.handler.ShouldThrottle(throttleErr) {
return throttleErr
}
rpcErr = throttleErr
return nil
}, backoff.NewExponentialBackOff(), c.handler.OnDoCommandError)
// RetryNotify gave up.
if throttleErr != nil {
return throttleErr
}
// check to see if we need to retry it.
if !c.checkForRetry(rpcErr) {
return rpcErr
}
}
}
func (b *BackoffReadCloser) Read(data []byte) (int, error) {
bytesRead := 0
var n int
var err error
backoff.RetryNotify(func() error {
n, err = b.reader.Read(data[bytesRead:])
bytesRead += n
if b.client.IsRetryable(err) {
return err
}
return nil
}, b.backoffConfig, func(err error, d time.Duration) {
protolion.Infof("Error reading (retrying): %#v", RetryError{
err: err.Error(),
timeTillNextRetry: d.String(),
bytesProcessed: bytesRead,
})
})
return bytesRead, err
}
// discover attempts to find new nodes in the cluster using the current nodes
func (c *Cluster) discover() {
// Keep retrying with exponential backoff.
b := backoff.NewExponentialBackOff()
// Never finish retrying (max interval is still 60s)
b.MaxElapsedTime = 0
// Keep trying to discover new nodes
for {
backoff.RetryNotify(func() error {
// If no hosts try seeding nodes
if len(c.GetNodes()) == 0 {
c.connectNodes(c.getSeeds())
}
return c.listenForNodeChanges()
}, b, func(err error, wait time.Duration) {
Log.Debugf("Error discovering hosts %s, waiting: %s", err, wait)
})
}
}
// DoCommand executes the specific rpc command wrapped in rpcFunc.
func (c *Connection) DoCommand(ctx context.Context, name string,
rpcFunc func(GenericClient) error) error {
for {
// we may or may not be in the process of reconnecting.
// if so we'll block here unless canceled by the caller.
connErr := c.waitForConnection(ctx)
if connErr != nil {
return connErr
}
var rpcErr error
// retry throttle errors w/backoff
throttleErr := backoff.RetryNotify(func() error {
rawClient := func() GenericClient {
c.mutex.Lock()
defer c.mutex.Unlock()
return c.client
}()
// try the rpc call, assuming that it exits
// immediately when ctx is canceled. will
// retry connectivity errors w/backoff.
throttleErr := rpcFunc(rawClient)
if throttleErr != nil && c.handler.ShouldRetry(name, throttleErr) {
return throttleErr
}
rpcErr = throttleErr
return nil
}, c.doCommandBackoff, c.handler.OnDoCommandError)
// RetryNotify gave up.
if throttleErr != nil {
return throttleErr
}
// check to see if we need to retry it.
if !c.checkForRetry(rpcErr) {
return rpcErr
}
}
}
// DoRequestWithExponentialBackOff makes an http request using the http.Client and http.Request associated with this context.
// You can pass a condition and a BackOff configuration. See https://github.com/cenkalti/backoff to know more about backoff.
// If no BackOff is provided it will use the default exponential BackOff configuration.
// See also ErrorIfStatusCodeIsNot function that provides a basic condition based on status code.
func (c *Context) DoRequestWithExponentialBackOff(condition BackoffCondition, b backoff.BackOff) (*http.Response, error) {
if b == nil {
b = backoff.NewExponentialBackOff()
}
err := backoff.RetryNotify(
func() error {
var err error
res, err := c.DoRequest()
if err != nil {
return err
}
return condition(res)
},
b,
func(err error, wait time.Duration) {
fmt.Println("˙\nBackoff:Waiting: ", wait)
fmt.Println(err)
fmt.Println("")
})
return c.Response(), err
}
func (w SlackWriter) Write(scanResults []models.ScanResult) error {
conf := config.Conf.Slack
for _, s := range scanResults {
channel := conf.Channel
if channel == "${servername}" {
channel = fmt.Sprintf("#%s", s.ServerName)
}
msg := message{
Text: msgText(s),
Username: conf.AuthUser,
IconEmoji: conf.IconEmoji,
Channel: channel,
Attachments: toSlackAttachments(s),
}
bytes, _ := json.Marshal(msg)
jsonBody := string(bytes)
f := func() (err error) {
resp, body, errs := gorequest.New().Proxy(config.Conf.HTTPProxy).Post(conf.HookURL).
Send(string(jsonBody)).End()
if resp.StatusCode != 200 {
log.Errorf("Resonse body: %s", body)
if len(errs) > 0 {
return errs[0]
}
}
return nil
}
notify := func(err error, t time.Duration) {
log.Warn("Retrying in ", t)
}
if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil {
return fmt.Errorf("HTTP Error: %s", err)
}
}
return nil
}
// doReconnect attempts a reconnection. It assumes that reconnectChan
// and reconnectErrPtr are the same ones in c, but are passed in to
// avoid having to take the mutex at the beginning of the method.
func (c *Connection) doReconnect(ctx context.Context, disconnectStatus DisconnectStatus,
reconnectChan chan struct{}, reconnectErrPtr *error) {
// inform the handler of our disconnected state
c.handler.OnDisconnected(ctx, disconnectStatus)
err := backoff.RetryNotify(func() error {
// try to connect
err := c.connect(ctx)
select {
case <-ctx.Done():
// context was canceled by Shutdown() or a user action
*reconnectErrPtr = ctx.Err()
// short-circuit Retry
return nil
default:
}
if !c.handler.ShouldRetryOnConnect(err) {
// A fatal error happened.
*reconnectErrPtr = err
// short-circuit Retry
return nil
}
return err
}, c.reconnectBackoff,
// give the caller a chance to log any other error or adjust state
c.handler.OnConnectError)
if err != nil {
// this shouldn't happen, but just in case.
*reconnectErrPtr = err
}
// close the reconnect channel to signal we're connected.
c.mutex.Lock()
defer c.mutex.Unlock()
close(reconnectChan)
c.reconnectChan = nil
c.cancelFunc = nil
c.reconnectErrPtr = nil
}
func getPort(ip, user, pass, id string, timeout time.Duration, c context.Context) (int, error) {
var port int
notify := func(e error, w time.Duration) {
logger.Errorf("Failed to get port from PIA (retry in %v): %v", w, e)
}
fn := func() error {
select {
default:
p, er := pia.RequestPort(ip, user, pass, id)
if er != nil {
return er
}
port = p
return nil
case <-c.Done():
return nil
}
}
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = timeout
return port, backoff.RetryNotify(fn, b, notify)
}
// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage) error {
config := api.DefaultConfig()
config.Address = provider.Endpoint
client, err := api.NewClient(config)
if err != nil {
return err
}
provider.client = client
go func() {
notify := func(err error, time time.Duration) {
log.Errorf("Consul connection error %+v, retrying in %s", err, time)
}
worker := func() error {
return provider.watch(configurationChan)
}
err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to consul server %+v", err)
}
}()
return err
}
func (a *apiServer) AddShard(shard uint64) error {
persistClient, err := a.getPersistClient()
if err != nil {
return err
}
ctx, cancel := context.WithCancel(context.Background())
a.shardCancelFuncsLock.Lock()
defer a.shardCancelFuncsLock.Unlock()
if _, ok := a.shardCancelFuncs[shard]; ok {
return fmt.Errorf("shard %d is being added twice; this is likely a bug", shard)
}
a.shardCancelFuncs[shard] = cancel
client, err := persistClient.SubscribePipelineInfos(ctx, &persist.SubscribePipelineInfosRequest{
IncludeInitial: true,
Shard: &persist.Shard{shard},
})
if err != nil {
return err
}
go func() {
for {
pipelineChange, err := client.Recv()
if err != nil {
return
}
if pipelineChange.Removed {
a.cancelFuncsLock.Lock()
cancel, ok := a.cancelFuncs[pipelineChange.Pipeline.PipelineName]
if ok {
cancel()
delete(a.cancelFuncs, pipelineChange.Pipeline.PipelineName)
} else {
protolion.Printf("trying to cancel a pipeline that we are not assigned to; this is likely a bug")
}
a.cancelFuncsLock.Unlock()
} else {
// We only want to start a goro to run the pipeline if the
// pipeline has more than one inputs
go func() {
b := backoff.NewExponentialBackOff()
// We set MaxElapsedTime to 0 because we want the retry to
// never stop.
// However, ideally we should crash this pps server so the
// pipeline gets reassigned to another pps server.
// The reason we don't do that right now is that pps and
// pfs are bundled together, so by crashing this program
// we will be crashing a pfs node too, which might cause
// cascading failures as other pps nodes might be depending
// on it.
b.MaxElapsedTime = 0
err = backoff.RetryNotify(func() error {
if err := a.runPipeline(newPipelineInfo(pipelineChange.Pipeline)); err != nil && !isContextCancelled(err) {
return err
}
return nil
}, b, func(err error, d time.Duration) {
if _, err = persistClient.UpdatePipelineState(context.Background(), &persist.UpdatePipelineStateRequest{
PipelineName: pipelineChange.Pipeline.PipelineName,
State: ppsclient.PipelineState_PIPELINE_RESTARTING,
RecentError: err.Error(),
}); err != nil {
protolion.Errorf("error updating pipeline state: %v", err)
}
})
// At this point we stop retrying and update the pipeline state
// to FAILED
if err != nil {
if _, err = persistClient.UpdatePipelineState(context.Background(), &persist.UpdatePipelineStateRequest{
PipelineName: pipelineChange.Pipeline.PipelineName,
State: ppsclient.PipelineState_PIPELINE_FAILURE,
RecentError: err.Error(),
}); err != nil {
protolion.Errorf("error updating pipeline state: %v", err)
}
}
}()
}
}
}()
return nil
}
func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool) error {
storeConfig := &store.Config{
ConnectionTimeout: 30 * time.Second,
Bucket: "traefik",
}
if provider.TLS != nil {
caPool := x509.NewCertPool()
if provider.TLS.CA != "" {
ca, err := ioutil.ReadFile(provider.TLS.CA)
if err != nil {
return fmt.Errorf("Failed to read CA. %s", err)
}
caPool.AppendCertsFromPEM(ca)
}
cert, err := tls.LoadX509KeyPair(provider.TLS.Cert, provider.TLS.Key)
if err != nil {
return fmt.Errorf("Failed to load keypair. %s", err)
}
storeConfig.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caPool,
InsecureSkipVerify: provider.TLS.InsecureSkipVerify,
}
}
operation := func() error {
kv, err := libkv.NewStore(
provider.storeType,
strings.Split(provider.Endpoint, ","),
storeConfig,
)
if err != nil {
return err
}
if _, err := kv.List(""); err != nil {
return err
}
provider.kvclient = kv
if provider.Watch {
pool.Go(func(stop chan bool) {
provider.watchKv(configurationChan, provider.Prefix, stop)
})
}
configuration := provider.loadConfig()
configurationChan <- types.ConfigMessage{
ProviderName: string(provider.storeType),
Configuration: configuration,
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("KV connection error %+v, retrying in %s", err, time)
}
err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
if err != nil {
log.Fatalf("Cannot connect to KV server %+v", err)
}
return nil
}