// Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications.
func Update() {
log.Info("updating vulnerabilities")
// Fetch updates.
status, responses := fetch()
// Merge responses.
vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses)
if err != nil {
log.Errorf("an error occured when merging update responses: %s", err)
return
}
responses = nil
// TODO(Quentin-M): Complete informations using NVD
// Insert packages.
log.Tracef("beginning insertion of %d packages for update", len(packages))
err = database.InsertPackages(packages)
if err != nil {
log.Errorf("an error occured when inserting packages for update: %s", err)
return
}
packages = nil
// Insert vulnerabilities.
log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
notifications, err := database.InsertVulnerabilities(vulnerabilities)
if err != nil {
log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
return
}
vulnerabilities = nil
// Insert notifications into the database.
err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
if err != nil {
log.Errorf("an error occured when inserting notifications for update: %s", err)
return
}
notifications = nil
// Update flags and notes.
for flagName, flagValue := range flags {
database.UpdateFlag(flagName, flagValue)
}
database.UpdateFlag(notesFlagName, notes)
// Update last successful update if every fetchers worked properly.
if status {
database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10))
}
log.Info("update finished")
}
// POSTVulnerabilities manually inserts a vulnerability into the database if it
// does not exist yet.
func POSTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
var parameters *database.AbstractVulnerability
if s, err := jsonhttp.ParseBody(r, ¶meters); err != nil {
jsonhttp.RenderError(w, s, err)
return
}
// Ensure that the vulnerability does not exist.
vulnerability, err := database.FindOneVulnerability(parameters.ID, []string{})
if err != nil && err != cerrors.ErrNotFound {
jsonhttp.RenderError(w, 0, err)
return
}
if vulnerability != nil {
jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("vulnerability already exists"))
return
}
// Insert packages.
packages := database.AbstractPackagesToPackages(parameters.AffectedPackages)
err = database.InsertPackages(packages)
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
var pkgNodes []string
for _, p := range packages {
pkgNodes = append(pkgNodes, p.Node)
}
// Insert vulnerability.
notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Insert notifications.
err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
jsonhttp.Render(w, http.StatusCreated, nil)
}
// PUTVulnerabilities updates a vulnerability if it exists.
func PUTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
var parameters *database.AbstractVulnerability
if s, err := httputils.ParseHTTPBody(r, ¶meters); err != nil {
httputils.WriteHTTPError(w, s, err)
return
}
parameters.ID = p.ByName("id")
// Ensure that the vulnerability exists.
_, err := database.FindOneVulnerability(parameters.ID, []string{})
if err != nil {
httputils.WriteHTTPError(w, 0, err)
return
}
// Insert packages.
packages := database.AbstractPackagesToPackages(parameters.AffectedPackages)
err = database.InsertPackages(packages)
if err != nil {
httputils.WriteHTTPError(w, 0, err)
return
}
var pkgNodes []string
for _, p := range packages {
pkgNodes = append(pkgNodes, p.Node)
}
// Insert vulnerability.
notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)})
if err != nil {
httputils.WriteHTTPError(w, 0, err)
return
}
// Insert notifications.
err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
if err != nil {
httputils.WriteHTTPError(w, 0, err)
return
}
httputils.WriteHTTP(w, http.StatusCreated, nil)
}
func findTask(whoAmI string, stopper *utils.Stopper) (string, database.Notification) {
for {
// Find a notification to send.
node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
if err != nil {
log.Warningf("could not get notification to send: %s", err)
}
// No notification or error: wait.
if notification == nil || err != nil {
if !stopper.Sleep(checkInterval) {
return "", nil
}
continue
}
// Lock the notification.
if hasLock, _ := database.Lock(node, lockDuration, whoAmI); hasLock {
log.Infof("found and locked a notification: %s", notification.GetName())
return node, notification
}
}
}
// Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications.
func Update() {
log.Info("updating vulnerabilities")
// Fetch updates in parallel.
var status = true
var responseC = make(chan *FetcherResponse, 0)
for n, f := range fetchers {
go func(name string, fetcher Fetcher) {
response, err := fetcher.FetchUpdate()
if err != nil {
log.Errorf("an error occured when fetching update '%s': %s.", name, err)
status = false
responseC <- nil
return
}
responseC <- &response
}(n, f)
}
// Collect results of updates.
var responses []*FetcherResponse
var notes []string
for i := 0; i < len(fetchers); {
select {
case resp := <-responseC:
if resp != nil {
responses = append(responses, resp)
notes = append(notes, resp.Notes...)
}
i++
}
}
close(responseC)
// TODO(Quentin-M): Merge responses together
// TODO(Quentin-M): Complete informations using NVD
// Store flags out of the response struct.
flags := make(map[string]string)
for _, response := range responses {
if response.FlagName != "" && response.FlagValue != "" {
flags[response.FlagName] = response.FlagValue
}
}
// Update health notes.
healthNotes = notes
// Build list of packages.
var packages []*database.Package
for _, response := range responses {
for _, v := range response.Vulnerabilities {
packages = append(packages, v.FixedIn...)
}
}
// Insert packages into the database.
log.Tracef("beginning insertion of %d packages for update", len(packages))
t := time.Now()
err := database.InsertPackages(packages)
log.Tracef("inserting %d packages took %v", len(packages), time.Since(t))
if err != nil {
log.Errorf("an error occured when inserting packages for update: %s", err)
updateHealth(false)
return
}
packages = nil
// Build a list of vulnerabilties.
var vulnerabilities []*database.Vulnerability
for _, response := range responses {
for _, v := range response.Vulnerabilities {
var packageNodes []string
for _, pkg := range v.FixedIn {
packageNodes = append(packageNodes, pkg.Node)
}
vulnerabilities = append(vulnerabilities, &database.Vulnerability{ID: v.ID, Link: v.Link, Priority: v.Priority, Description: v.Description, FixedInNodes: packageNodes})
}
}
responses = nil
// Insert vulnerabilities into the database.
log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
t = time.Now()
notifications, err := database.InsertVulnerabilities(vulnerabilities)
log.Tracef("inserting %d vulnerabilities took %v", len(vulnerabilities), time.Since(t))
if err != nil {
log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
updateHealth(false)
return
}
vulnerabilities = nil
// Insert notifications into the database.
err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
if err != nil {
log.Errorf("an error occured when inserting notifications for update: %s", err)
updateHealth(false)
return
}
notifications = nil
// Update flags in the database.
for flagName, flagValue := range flags {
database.UpdateFlag(flagName, flagValue)
}
// Update health depending on the status of the fetchers.
updateHealth(status)
if status {
now := time.Now().UTC()
database.UpdateFlag(flagName, strconv.FormatInt(now.Unix(), 10))
healthLatestSuccessfulUpdate = now
}
log.Info("update finished")
}
// Run pops notifications from the database, lock them, send them, mark them as
// send and unlock them
//
// It uses an exponential backoff when POST requests fail
func (notifier *HTTPNotifier) Run(st *utils.Stopper) {
defer st.End()
whoAmI := uuid.New()
log.Infof("HTTP notifier started. URL: %s. Lock Identifier: %s", notifier.url, whoAmI)
for {
node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
if notification == nil || err != nil {
if err != nil {
log.Warningf("could not get notification to send: %s.", err)
}
if !st.Sleep(checkInterval) {
break
}
continue
}
// Try to lock the notification
hasLock, hasLockUntil := database.Lock(node, lockDuration, whoAmI)
if !hasLock {
continue
}
for backOff := time.Duration(0); ; backOff = timeutil.ExpBackoff(backOff, maxBackOff) {
// Backoff, it happens when an error occurs during the communication
// with the notification endpoint
if backOff > 0 {
// Renew lock before going to sleep if necessary
if time.Now().Add(backOff).After(hasLockUntil.Add(-refreshLockAnticipation)) {
hasLock, hasLockUntil = database.Lock(node, lockDuration, whoAmI)
if !hasLock {
log.Warning("lost lock ownership, aborting")
break
}
}
// Sleep
if !st.Sleep(backOff) {
return
}
}
// Get notification content
content, err := notification.GetContent()
if err != nil {
log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Marshal the notification content
jsonContent, err := json.Marshal(struct {
Name, Type string
Content interface{}
}{
Name: notification.GetName(),
Type: notification.GetType(),
Content: content,
})
if err != nil {
log.Errorf("could not marshal content of notification '%s': %s", notification.GetName(), err.Error())
break
}
// Post notification
req, _ := http.NewRequest("POST", notifier.url, bytes.NewBuffer(jsonContent))
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
log.Warningf("could not post notification '%s': %s", notification.GetName(), err.Error())
continue
}
res.Body.Close()
if res.StatusCode != 200 && res.StatusCode != 201 {
log.Warningf("could not post notification '%s': got status code %d", notification.GetName(), res.StatusCode)
continue
}
// Mark the notification as sent
database.MarkNotificationAsSent(node)
log.Infof("sent notification '%s' successfully", notification.GetName())
break
}
if hasLock {
database.Unlock(node, whoAmI)
}
}
log.Info("HTTP notifier stopped")
}