func Run(config *config.APIConfig, ctx *context.RouteContext, st *utils.Stopper) {
defer st.End()
// Do not run the API service if there is no config.
if config == nil {
log.Infof("main API service is disabled.")
return
}
log.Infof("starting main API on port %d.", config.Port)
tlsConfig, err := tlsClientConfig(config.CAFile)
if err != nil {
log.Fatalf("could not initialize client cert authentication: %s\n", err)
}
if tlsConfig != nil {
log.Info("main API configured with client certificate authentication")
}
srv := &graceful.Server{
Timeout: 0, // Already handled by our TimeOut middleware
NoSignalHandling: true, // We want to use our own Stopper
Server: &http.Server{
Addr: ":" + strconv.Itoa(config.Port),
TLSConfig: tlsConfig,
Handler: http.TimeoutHandler(newAPIHandler(ctx), config.Timeout, timeoutResponse),
},
}
listenAndServeWithStopper(srv, st, config.CertFile, config.KeyFile)
log.Info("main API stopped")
}
func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *database.VulnerabilityNotification {
for {
// Find a notification to send.
notification, err := datastore.GetAvailableNotification(renotifyInterval)
if err != nil {
// There is no notification or an error occurred.
if err != cerrors.ErrNotFound {
log.Warningf("could not get notification to send: %s", err)
}
// Wait.
if !stopper.Sleep(checkInterval) {
return nil
}
continue
}
// Lock the notification.
if hasLock, _ := datastore.Lock(notification.Name, whoAmI, lockDuration, false); hasLock {
log.Infof("found and locked a notification: %s", notification.Name)
return ¬ification
}
}
}
func handleTask(notification database.Notification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Get notification content.
// TODO(Quentin-M): Split big notifications.
notificationContent, err := notification.GetContent()
if err != nil {
log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err)
return false, false
}
// Create notification.
payload := &Notification{
Name: notification.GetName(),
Type: notification.GetType(),
Content: notificationContent,
}
// Send notification.
for notifierName, notifier := range notifiers {
var attempts int
var backOff time.Duration
for {
// Max attempts exceeded.
if attempts >= maxAttempts {
log.Infof("giving up on sending notification '%s' to notifier '%s': max attempts exceeded (%d)\n", notification.GetName(), notifierName, maxAttempts)
return false, false
}
// Backoff.
if backOff > 0 {
log.Infof("waiting %v before retrying to send notification '%s' to notifier '%s' (Attempt %d / %d)\n", backOff, notification.GetName(), notifierName, attempts+1, maxAttempts)
if !st.Sleep(backOff) {
return false, true
}
}
// Send using the current notifier.
if err := notifier.Send(payload); err == nil {
// Send has been successful. Go to the next one.
break
}
// Send failed; increase attempts/backoff and retry.
log.Errorf("could not send notification '%s' to notifier '%s': %s", notification.GetName(), notifierName, err)
backOff = timeutil.ExpBackoff(backOff, maxBackOff)
attempts++
}
}
log.Infof("successfully sent notification '%s'\n", notification.GetName())
return true, false
}
// RunHealth launches the Health API, which only exposes a method to fetch
// clair's health without any security or authentification mechanism.
func RunHealth(port int, st *utils.Stopper) {
log.Infof("starting Health API on port %d.", port)
defer func() {
log.Info("Health API stopped")
st.End()
}()
srv := &graceful.Server{
Timeout: 10 * time.Second, // Interrupt health checks when stopping
NoSignalHandling: true, // We want to use our own Stopper
Server: &http.Server{
Addr: ":" + strconv.Itoa(port),
Handler: NewHealthRouter(),
},
}
listenAndServeWithStopper(srv, st, "", "")
}
// listenAndServeWithStopper wraps graceful.Server's
// ListenAndServe/ListenAndServeTLS and adds the ability to interrupt them with
// the provided utils.Stopper
func listenAndServeWithStopper(srv *graceful.Server, st *utils.Stopper, certFile, keyFile string) {
go func() {
<-st.Chan()
srv.Stop(0)
}()
var err error
if certFile != "" && keyFile != "" {
log.Info("API: TLS Enabled")
err = srv.ListenAndServeTLS(certFile, keyFile)
} else {
err = srv.ListenAndServe()
}
if opErr, ok := err.(*net.OpError); !ok || (ok && opErr.Op != "accept") {
log.Fatal(err)
}
}
// RunMain launches the main API, which exposes every possible interactions
// with clair.
func RunMain(conf *Config, st *utils.Stopper) {
log.Infof("starting API on port %d.", conf.Port)
defer func() {
log.Info("API stopped")
st.End()
}()
srv := &graceful.Server{
Timeout: 0, // Already handled by our TimeOut middleware
NoSignalHandling: true, // We want to use our own Stopper
Server: &http.Server{
Addr: ":" + strconv.Itoa(conf.Port),
TLSConfig: setupClientCert(conf.CAFile),
Handler: NewVersionRouter(conf.TimeOut),
},
}
listenAndServeWithStopper(srv, st, conf.CertFile, conf.KeyFile)
}
// RunHealth launches the Health API, which only exposes a method to fetch
// Clair's health without any security or authentication mechanism.
func RunHealth(config *config.APIConfig, st *utils.Stopper) {
defer st.End()
// Do not run the API service if there is no config.
if config == nil {
log.Infof("health API service is disabled.")
return
}
log.Infof("starting health API on port %d.", config.HealthPort)
srv := &graceful.Server{
Timeout: 10 * time.Second, // Interrupt health checks when stopping
NoSignalHandling: true, // We want to use our own Stopper
Server: &http.Server{
Addr: ":" + strconv.Itoa(config.HealthPort),
Handler: NewHealthRouter(),
},
}
listenAndServeWithStopper(srv, st, "", "")
log.Info("health API stopped")
}
func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Send notification.
for notifierName, notifier := range notifiers {
var attempts int
var backOff time.Duration
for {
// Max attempts exceeded.
if attempts >= maxAttempts {
log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts)
return false, false
}
// Backoff.
if backOff > 0 {
log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts)
if !st.Sleep(backOff) {
return false, true
}
}
// Send using the current notifier.
if err := notifier.Send(notification); err != nil {
// Send failed; increase attempts/backoff and retry.
promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc()
log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err)
backOff = timeutil.ExpBackoff(backOff, maxBackOff)
attempts++
continue
}
// Send has been successful. Go to the next notifier.
break
}
}
log.Infof("successfully sent notification '%s'\n", notification.Name)
return true, false
}
func findTask(whoAmI string, stopper *utils.Stopper) (string, database.Notification) {
for {
// Find a notification to send.
node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
if err != nil {
log.Warningf("could not get notification to send: %s", err)
}
// No notification or error: wait.
if notification == nil || err != nil {
if !stopper.Sleep(checkInterval) {
return "", nil
}
continue
}
// Lock the notification.
if hasLock, _ := database.Lock(node, lockDuration, whoAmI); hasLock {
log.Infof("found and locked a notification: %s", notification.GetName())
return node, notification
}
}
}
// Run updates the vulnerability database at regular intervals
func Run(interval time.Duration, st *utils.Stopper) {
defer st.End()
// Do not run the updater if the interval is 0
if interval == 0 {
log.Infof("updater service is disabled.")
return
}
whoAmI := uuid.New()
healthIdentifier = whoAmI
log.Infof("updater service started. lock identifier: %s", whoAmI)
for {
// Set the next update time to (last update time + interval) or now if there
// is no last update time stored in database (first update) or if an error
// occurs
nextUpdate := time.Now().UTC()
if lastUpdateTSS, err := database.GetFlagValue(flagName); err == nil && lastUpdateTSS != "" {
if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil {
healthLatestSuccessfulUpdate = time.Unix(lastUpdateTS, 0)
nextUpdate = time.Unix(lastUpdateTS, 0).Add(interval)
}
}
// If the next update timer is in the past, then try to update.
if nextUpdate.Before(time.Now().UTC()) {
// Attempt to get a lock on the the update.
log.Debug("attempting to obtain update lock")
hasLock, hasLockUntil := database.Lock(flagName, lockDuration, whoAmI)
if hasLock {
healthLockOwner = healthIdentifier
// Launch update in a new go routine.
doneC := make(chan bool, 1)
go func() {
Update()
doneC <- true
}()
for done := false; !done; {
select {
case <-doneC:
done = true
case <-time.After(refreshLockDuration):
// Refresh the lock until the update is done.
database.Lock(flagName, lockDuration, whoAmI)
}
}
// Unlock the update.
database.Unlock(flagName, whoAmI)
} else {
lockOwner, lockExpiration, err := database.LockInfo(flagName)
if err != nil {
log.Debug("update lock is already taken")
nextUpdate = hasLockUntil
} else {
log.Debugf("update lock is already taken by %s until %v", lockOwner, lockExpiration)
nextUpdate = lockExpiration
healthLockOwner = lockOwner
}
}
}
// Sleep, but remain stoppable until approximately the next update time.
now := time.Now().UTC()
waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
log.Debugf("next update attempt scheduled for %v.", waitUntil)
if !waitUntil.Before(now) {
if !st.Sleep(waitUntil.Sub(time.Now())) {
break
}
}
}
log.Info("updater service stopped")
}
// Run starts the Notifier service.
func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) {
defer stopper.End()
// Configure registered notifiers.
for notifierName, notifier := range notifiers {
if configured, err := notifier.Configure(config); configured {
log.Infof("notifier '%s' configured\n", notifierName)
} else {
delete(notifiers, notifierName)
if err != nil {
log.Errorf("could not configure notifier '%s': %s", notifierName, err)
}
}
}
// Do not run the updater if there is no notifier enabled.
if len(notifiers) == 0 {
log.Infof("notifier service is disabled")
return
}
whoAmI := uuid.New()
log.Infof("notifier service started. lock identifier: %s\n", whoAmI)
for running := true; running; {
// Find task.
notification := findTask(datastore, config.RenotifyInterval, whoAmI, stopper)
if notification == nil {
// Interrupted while finding a task, Clair is stopping.
break
}
// Handle task.
done := make(chan bool, 1)
go func() {
success, interrupted := handleTask(*notification, stopper, config.Attempts)
if success {
utils.PrometheusObserveTimeMilliseconds(promNotifierLatencyMilliseconds, notification.Created)
datastore.SetNotificationNotified(notification.Name)
}
if interrupted {
running = false
}
datastore.Unlock(notification.Name, whoAmI)
done <- true
}()
// Refresh task lock until done.
outer:
for {
select {
case <-done:
break outer
case <-time.After(refreshLockDuration):
datastore.Lock(notification.Name, whoAmI, lockDuration, true)
}
}
}
log.Info("notifier service stopped")
}
// Run updates the vulnerability database at regular intervals.
func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) {
defer st.End()
// Do not run the updater if there is no config or if the interval is 0.
if config == nil || config.Interval == 0 {
log.Infof("updater service is disabled.")
return
}
whoAmI := uuid.New()
log.Infof("updater service started. lock identifier: %s", whoAmI)
for {
var stop bool
// Determine if this is the first update and define the next update time.
// The next update time is (last update time + interval) or now if this is the first update.
nextUpdate := time.Now().UTC()
lastUpdate, firstUpdate, err := getLastUpdate(datastore)
if err != nil {
log.Errorf("an error occured while getting the last update time")
nextUpdate = nextUpdate.Add(config.Interval)
} else if firstUpdate == false {
nextUpdate = lastUpdate.Add(config.Interval)
}
// If the next update timer is in the past, then try to update.
if nextUpdate.Before(time.Now().UTC()) {
// Attempt to get a lock on the the update.
log.Debug("attempting to obtain update lock")
hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false)
if hasLock {
// Launch update in a new go routine.
doneC := make(chan bool, 1)
go func() {
Update(datastore, firstUpdate)
doneC <- true
}()
for done := false; !done && !stop; {
select {
case <-doneC:
done = true
case <-time.After(refreshLockDuration):
// Refresh the lock until the update is done.
datastore.Lock(lockName, whoAmI, lockDuration, true)
case <-st.Chan():
stop = true
}
}
// Unlock the update.
datastore.Unlock(lockName, whoAmI)
if stop {
break
}
continue
} else {
lockOwner, lockExpiration, err := datastore.FindLock(lockName)
if err != nil {
log.Debug("update lock is already taken")
nextUpdate = hasLockUntil
} else {
log.Debugf("update lock is already taken by %s until %v", lockOwner, lockExpiration)
nextUpdate = lockExpiration
}
}
}
// Sleep, but remain stoppable until approximately the next update time.
now := time.Now().UTC()
waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
log.Debugf("next update attempt scheduled for %v.", waitUntil)
if !waitUntil.Before(now) {
if !st.Sleep(waitUntil.Sub(time.Now())) {
break
}
}
}
// Clean resources.
for _, metadataFetcher := range metadataFetchers {
metadataFetcher.Clean()
}
for _, fetcher := range fetchers {
fetcher.Clean()
}
log.Info("updater service stopped")
}
// Run updates the vulnerability database at regular intervals.
func Run(config *config.UpdaterConfig, st *utils.Stopper) {
defer st.End()
// Do not run the updater if there is no config or if the interval is 0.
if config == nil || config.Interval == 0 {
log.Infof("updater service is disabled.")
return
}
// Register healthchecker.
health.RegisterHealthchecker("updater", Healthcheck)
whoAmI := uuid.New()
log.Infof("updater service started. lock identifier: %s", whoAmI)
for {
// Set the next update time to (last update time + interval) or now if there
// is no last update time stored in database (first update) or if an error
// occurs.
var nextUpdate time.Time
var stop bool
if lastUpdate := getLastUpdate(); !lastUpdate.IsZero() {
nextUpdate = lastUpdate.Add(config.Interval)
} else {
nextUpdate = time.Now().UTC()
}
// If the next update timer is in the past, then try to update.
if nextUpdate.Before(time.Now().UTC()) {
// Attempt to get a lock on the the update.
log.Debug("attempting to obtain update lock")
hasLock, hasLockUntil := database.Lock(flagName, lockDuration, whoAmI)
if hasLock {
// Launch update in a new go routine.
doneC := make(chan bool, 1)
go func() {
Update()
doneC <- true
}()
for done := false; !done && !stop; {
select {
case <-doneC:
done = true
case <-time.After(refreshLockDuration):
// Refresh the lock until the update is done.
database.Lock(flagName, lockDuration, whoAmI)
case <-st.Chan():
stop = true
}
}
// Unlock the update.
database.Unlock(flagName, whoAmI)
if stop {
break
}
continue
} else {
lockOwner, lockExpiration, err := database.LockInfo(flagName)
if err != nil {
log.Debug("update lock is already taken")
nextUpdate = hasLockUntil
} else {
log.Debugf("update lock is already taken by %s until %v", lockOwner, lockExpiration)
nextUpdate = lockExpiration
}
}
}
// Sleep, but remain stoppable until approximately the next update time.
now := time.Now().UTC()
waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
log.Debugf("next update attempt scheduled for %v.", waitUntil)
if !waitUntil.Before(now) {
if !st.Sleep(waitUntil.Sub(time.Now())) {
break
}
}
}
log.Info("updater service stopped")
}
// Run pops notifications from the database, lock them, send them, mark them as
// send and unlock them
//
// It uses an exponential backoff when POST requests fail
func (notifier *HTTPNotifier) Run(st *utils.Stopper) {
defer st.End()
whoAmI := uuid.New()
log.Infof("HTTP notifier started. URL: %s. Lock Identifier: %s", notifier.url, whoAmI)
for {
node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
if notification == nil || err != nil {
if err != nil {
log.Warningf("could not get notification to send: %s.", err)
}
if !st.Sleep(checkInterval) {
break
}
continue
}
// Try to lock the notification
hasLock, hasLockUntil := database.Lock(node, lockDuration, whoAmI)
if !hasLock {
continue
}
for backOff := time.Duration(0); ; backOff = timeutil.ExpBackoff(backOff, maxBackOff) {
// Backoff, it happens when an error occurs during the communication
// with the notification endpoint
if backOff > 0 {
// Renew lock before going to sleep if necessary
if time.Now().Add(backOff).After(hasLockUntil.Add(-refreshLockAnticipation)) {
hasLock, hasLockUntil = database.Lock(node, lockDuration, whoAmI)
if !hasLock {
log.Warning("lost lock ownership, aborting")
break
}
}
// Sleep
if !st.Sleep(backOff) {
return
}
}
// Get notification content
content, err := notification.GetContent()
if err != nil {
log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Marshal the notification content
jsonContent, err := json.Marshal(struct {
Name, Type string
Content interface{}
}{
Name: notification.GetName(),
Type: notification.GetType(),
Content: content,
})
if err != nil {
log.Errorf("could not marshal content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Post notification
req, _ := http.NewRequest("POST", notifier.url, bytes.NewBuffer(jsonContent))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
log.Warningf("could not post notification '%s': %s", notification.GetName(), err.Error())
continue
}
res.Body.Close()
if res.StatusCode != 200 && res.StatusCode != 201 {
log.Warningf("could not post notification '%s': got status code %d", notification.GetName(), res.StatusCode)
continue
}
// Mark the notification as sent
database.MarkNotificationAsSent(node)
log.Infof("sent notification '%s' successfully", notification.GetName())
break
}
if hasLock {
database.Unlock(node, whoAmI)
}
}
log.Info("HTTP notifier stopped")
}
// Run starts the Notifier service.
func Run(config *config.NotifierConfig, stopper *utils.Stopper) {
defer stopper.End()
// Configure registered notifiers.
for notifierName, notifier := range notifiers {
if configured, err := notifier.Configure(config); configured {
log.Infof("notifier '%s' configured\n", notifierName)
} else {
delete(notifiers, notifierName)
if err != nil {
log.Errorf("could not configure notifier '%s': %s", notifierName, err)
}
}
}
// Do not run the updater if there is no notifier enabled.
if len(notifiers) == 0 {
log.Infof("notifier service is disabled")
return
}
whoAmI := uuid.New()
log.Infof("notifier service started. lock identifier: %s\n", whoAmI)
// Register healthchecker.
health.RegisterHealthchecker("notifier", Healthcheck)
for running := true; running; {
// Find task.
// TODO(Quentin-M): Combine node and notification.
node, notification := findTask(whoAmI, stopper)
if node == "" && notification == nil {
// Interrupted while finding a task, Clair is stopping.
break
}
// Handle task.
done := make(chan bool, 1)
go func() {
success, interrupted := handleTask(notification, stopper, config.Attempts)
if success {
database.MarkNotificationAsSent(node)
}
if interrupted {
running = false
}
database.Unlock(node, whoAmI)
done <- true
}()
// Refresh task lock until done.
outer:
for {
select {
case <-done:
break outer
case <-time.After(refreshLockDuration):
database.Lock(node, lockDuration, whoAmI)
}
}
}
log.Info("notifier service stopped")
}