func (r *pcsStepRetry) step(fn pcsStepFunc) (next pcsStepper) {
ttl, err := fn()
if err == nil {
next = &pcsStepNext{aft: ttl}
} else {
next = &pcsStepRetry{aft: timeutil.ExpBackoff(r.aft, time.Minute)}
log.Printf("go-oidc: provider config sync falied, retyring in %v: %v", next.after(), err)
}
return
}
func (r *pcsStepRetry) step(fn pcsStepFunc) (next pcsStepper) {
ttl, err := fn()
if err == nil {
next = &pcsStepNext{aft: ttl}
log.Infof("Provider config sync no longer failing")
} else {
next = &pcsStepRetry{aft: timeutil.ExpBackoff(r.aft, time.Minute)}
log.Errorf("Provider config sync still failing, retrying in %v: %v", next.after(), err)
}
return
}
func handleTask(notification database.Notification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Get notification content.
// TODO(Quentin-M): Split big notifications.
notificationContent, err := notification.GetContent()
if err != nil {
log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err)
return false, false
}
// Create notification.
payload := &Notification{
Name: notification.GetName(),
Type: notification.GetType(),
Content: notificationContent,
}
// Send notification.
for notifierName, notifier := range notifiers {
var attempts int
var backOff time.Duration
for {
// Max attempts exceeded.
if attempts >= maxAttempts {
log.Infof("giving up on sending notification '%s' to notifier '%s': max attempts exceeded (%d)\n", notification.GetName(), notifierName, maxAttempts)
return false, false
}
// Backoff.
if backOff > 0 {
log.Infof("waiting %v before retrying to send notification '%s' to notifier '%s' (Attempt %d / %d)\n", backOff, notification.GetName(), notifierName, attempts+1, maxAttempts)
if !st.Sleep(backOff) {
return false, true
}
}
// Send using the current notifier.
if err := notifier.Send(payload); err == nil {
// Send has been successful. Go to the next one.
break
}
// Send failed; increase attempts/backoff and retry.
log.Errorf("could not send notification '%s' to notifier '%s': %s", notification.GetName(), notifierName, err)
backOff = timeutil.ExpBackoff(backOff, maxBackOff)
attempts++
}
}
log.Infof("successfully sent notification '%s'\n", notification.GetName())
return true, false
}
func waitForProviderConfig(hc phttp.Client, issuerURL string, clock clockwork.Clock) (pcfg ProviderConfig) {
var sleep time.Duration
var err error
for {
pcfg, err = FetchProviderConfig(hc, issuerURL)
if err == nil {
break
}
sleep = timeutil.ExpBackoff(sleep, time.Minute)
fmt.Printf("Failed fetching provider config, trying again in %v: %v\n", sleep, err)
time.Sleep(sleep)
}
return
}
func (r *PrivateKeyRotator) Run() chan struct{} {
attempt := func() {
k, err := r.generateKey()
if err != nil {
log.Errorf("Failed generating signing key: %v", err)
return
}
exp := r.expiresAt()
if err := rotatePrivateKeys(r.repo, k, r.keep, exp); err != nil {
log.Errorf("Failed key rotation: %v", err)
return
}
log.Infof("Rotated signing keys: id=%s expiresAt=%s", k.ID(), exp)
}
stop := make(chan struct{})
go func() {
for {
var nextRotation time.Duration
var sleep time.Duration
var err error
for {
if nextRotation, err = r.nextRotation(); err == nil {
break
}
sleep = ptime.ExpBackoff(sleep, time.Minute)
log.Errorf("error getting nextRotation, retrying in %v: %v", sleep, err)
time.Sleep(sleep)
}
log.Infof("will rotate keys in %v", nextRotation)
select {
case <-r.clock.After(nextRotation):
attempt()
case <-stop:
return
}
}
}()
return stop
}
func (r *PrivateKeyRotator) Run() chan struct{} {
attempt := func() {
k, err := r.generateKey()
if err != nil {
log.Printf("go-oidc: failed generating signing key: %v", err)
return
}
exp := r.expiresAt()
if err := rotatePrivateKeys(r.repo, k, r.keep, exp); err != nil {
log.Printf("go-oidc: key rotation failed: %v", err)
return
}
}
stop := make(chan struct{})
go func() {
for {
var nextRotation time.Duration
var sleep time.Duration
var err error
for {
if nextRotation, err = r.nextRotation(); err == nil {
break
}
sleep = ptime.ExpBackoff(sleep, time.Minute)
log.Printf("go-oidc: error getting nextRotation, retrying in %v: %v", sleep, err)
time.Sleep(sleep)
}
select {
case <-r.clock.After(nextRotation):
attempt()
case <-stop:
return
}
}
}()
return stop
}
func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Send notification.
for notifierName, notifier := range notifiers {
var attempts int
var backOff time.Duration
for {
// Max attempts exceeded.
if attempts >= maxAttempts {
log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts)
return false, false
}
// Backoff.
if backOff > 0 {
log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts)
if !st.Sleep(backOff) {
return false, true
}
}
// Send using the current notifier.
if err := notifier.Send(notification); err != nil {
// Send failed; increase attempts/backoff and retry.
promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc()
log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err)
backOff = timeutil.ExpBackoff(backOff, maxBackOff)
attempts++
continue
}
// Send has been successful. Go to the next notifier.
break
}
}
log.Infof("successfully sent notification '%s'\n", notification.Name)
return true, false
}
func (s *KeySetSyncer) Run() chan struct{} {
stop := make(chan struct{})
go func() {
var failing bool
var next time.Duration
for {
exp, err := sync(s.readable, s.writable, s.clock)
if err != nil || exp == 0 {
if !failing {
failing = true
next = time.Second
} else {
next = timeutil.ExpBackoff(next, time.Minute)
}
if exp == 0 {
log.Errorf("Synced to already expired key set, retrying in %v: %v", next, err)
} else {
log.Errorf("Failed syncing key set, retrying in %v: %v", next, err)
}
} else {
failing = false
next = exp / 2
log.Infof("Synced key set, checking again in %v", next)
}
select {
case <-s.clock.After(next):
continue
case <-stop:
return
}
}
}()
return stop
}
// Run pops notifications from the database, lock them, send them, mark them as
// send and unlock them
//
// It uses an exponential backoff when POST requests fail
func (notifier *HTTPNotifier) Run(st *utils.Stopper) {
defer st.End()
whoAmI := uuid.New()
log.Infof("HTTP notifier started. URL: %s. Lock Identifier: %s", notifier.url, whoAmI)
for {
node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
if notification == nil || err != nil {
if err != nil {
log.Warningf("could not get notification to send: %s.", err)
}
if !st.Sleep(checkInterval) {
break
}
continue
}
// Try to lock the notification
hasLock, hasLockUntil := database.Lock(node, lockDuration, whoAmI)
if !hasLock {
continue
}
for backOff := time.Duration(0); ; backOff = timeutil.ExpBackoff(backOff, maxBackOff) {
// Backoff, it happens when an error occurs during the communication
// with the notification endpoint
if backOff > 0 {
// Renew lock before going to sleep if necessary
if time.Now().Add(backOff).After(hasLockUntil.Add(-refreshLockAnticipation)) {
hasLock, hasLockUntil = database.Lock(node, lockDuration, whoAmI)
if !hasLock {
log.Warning("lost lock ownership, aborting")
break
}
}
// Sleep
if !st.Sleep(backOff) {
return
}
}
// Get notification content
content, err := notification.GetContent()
if err != nil {
log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Marshal the notification content
jsonContent, err := json.Marshal(struct {
Name, Type string
Content interface{}
}{
Name: notification.GetName(),
Type: notification.GetType(),
Content: content,
})
if err != nil {
log.Errorf("could not marshal content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Post notification
req, _ := http.NewRequest("POST", notifier.url, bytes.NewBuffer(jsonContent))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
log.Warningf("could not post notification '%s': %s", notification.GetName(), err.Error())
continue
}
res.Body.Close()
if res.StatusCode != 200 && res.StatusCode != 201 {
log.Warningf("could not post notification '%s': got status code %d", notification.GetName(), res.StatusCode)
continue
}
// Mark the notification as sent
database.MarkNotificationAsSent(node)
log.Infof("sent notification '%s' successfully", notification.GetName())
break
}
if hasLock {
database.Unlock(node, whoAmI)
}
}
log.Info("HTTP notifier stopped")
}
