Beispiel #1
0
// Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications.
func Update() {
	log.Info("updating vulnerabilities")

	// Fetch updates.
	status, responses := fetch()

	// Merge responses.
	vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses)
	if err != nil {
		log.Errorf("an error occured when merging update responses: %s", err)
		return
	}
	responses = nil

	// TODO(Quentin-M): Complete informations using NVD

	// Insert packages.
	log.Tracef("beginning insertion of %d packages for update", len(packages))
	err = database.InsertPackages(packages)
	if err != nil {
		log.Errorf("an error occured when inserting packages for update: %s", err)
		return
	}
	packages = nil

	// Insert vulnerabilities.
	log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
	notifications, err := database.InsertVulnerabilities(vulnerabilities)
	if err != nil {
		log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
		return
	}
	vulnerabilities = nil

	// Insert notifications into the database.
	err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
	if err != nil {
		log.Errorf("an error occured when inserting notifications for update: %s", err)
		return
	}
	notifications = nil

	// Update flags and notes.
	for flagName, flagValue := range flags {
		database.UpdateFlag(flagName, flagValue)
	}
	database.UpdateFlag(notesFlagName, notes)

	// Update last successful update if every fetchers worked properly.
	if status {
		database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10))
	}
	log.Info("update finished")
}
Beispiel #2
0
// POSTVulnerabilities manually inserts a vulnerability into the database if it
// does not exist yet.
func POSTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	var parameters *database.AbstractVulnerability
	if s, err := jsonhttp.ParseBody(r, &parameters); err != nil {
		jsonhttp.RenderError(w, s, err)
		return
	}

	// Ensure that the vulnerability does not exist.
	vulnerability, err := database.FindOneVulnerability(parameters.ID, []string{})
	if err != nil && err != cerrors.ErrNotFound {
		jsonhttp.RenderError(w, 0, err)
		return
	}
	if vulnerability != nil {
		jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("vulnerability already exists"))
		return
	}

	// Insert packages.
	packages := database.AbstractPackagesToPackages(parameters.AffectedPackages)
	err = database.InsertPackages(packages)
	if err != nil {
		jsonhttp.RenderError(w, 0, err)
		return
	}
	var pkgNodes []string
	for _, p := range packages {
		pkgNodes = append(pkgNodes, p.Node)
	}

	// Insert vulnerability.
	notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)})
	if err != nil {
		jsonhttp.RenderError(w, 0, err)
		return
	}

	// Insert notifications.
	err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
	if err != nil {
		jsonhttp.RenderError(w, 0, err)
		return
	}

	jsonhttp.Render(w, http.StatusCreated, nil)
}
Beispiel #3
0
// PUTVulnerabilities updates a vulnerability if it exists.
func PUTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	var parameters *database.AbstractVulnerability
	if s, err := httputils.ParseHTTPBody(r, &parameters); err != nil {
		httputils.WriteHTTPError(w, s, err)
		return
	}
	parameters.ID = p.ByName("id")

	// Ensure that the vulnerability exists.
	_, err := database.FindOneVulnerability(parameters.ID, []string{})
	if err != nil {
		httputils.WriteHTTPError(w, 0, err)
		return
	}

	// Insert packages.
	packages := database.AbstractPackagesToPackages(parameters.AffectedPackages)
	err = database.InsertPackages(packages)
	if err != nil {
		httputils.WriteHTTPError(w, 0, err)
		return
	}
	var pkgNodes []string
	for _, p := range packages {
		pkgNodes = append(pkgNodes, p.Node)
	}

	// Insert vulnerability.
	notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)})
	if err != nil {
		httputils.WriteHTTPError(w, 0, err)
		return
	}

	// Insert notifications.
	err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
	if err != nil {
		httputils.WriteHTTPError(w, 0, err)
		return
	}

	httputils.WriteHTTP(w, http.StatusCreated, nil)
}
Beispiel #4
0
func findTask(whoAmI string, stopper *utils.Stopper) (string, database.Notification) {
	for {
		// Find a notification to send.
		node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
		if err != nil {
			log.Warningf("could not get notification to send: %s", err)
		}

		// No notification or error: wait.
		if notification == nil || err != nil {
			if !stopper.Sleep(checkInterval) {
				return "", nil
			}
			continue
		}

		// Lock the notification.
		if hasLock, _ := database.Lock(node, lockDuration, whoAmI); hasLock {
			log.Infof("found and locked a notification: %s", notification.GetName())
			return node, notification
		}
	}
}
Beispiel #5
0
// Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications.
func Update() {
	log.Info("updating vulnerabilities")

	// Fetch updates in parallel.
	var status = true
	var responseC = make(chan *FetcherResponse, 0)
	for n, f := range fetchers {
		go func(name string, fetcher Fetcher) {
			response, err := fetcher.FetchUpdate()
			if err != nil {
				log.Errorf("an error occured when fetching update '%s': %s.", name, err)
				status = false
				responseC <- nil
				return
			}

			responseC <- &response
		}(n, f)
	}

	// Collect results of updates.
	var responses []*FetcherResponse
	var notes []string
	for i := 0; i < len(fetchers); {
		select {
		case resp := <-responseC:
			if resp != nil {
				responses = append(responses, resp)
				notes = append(notes, resp.Notes...)
			}
			i++
		}
	}

	close(responseC)

	// TODO(Quentin-M): Merge responses together
	// TODO(Quentin-M): Complete informations using NVD

	// Store flags out of the response struct.
	flags := make(map[string]string)
	for _, response := range responses {
		if response.FlagName != "" && response.FlagValue != "" {
			flags[response.FlagName] = response.FlagValue
		}
	}

	// Update health notes.
	healthNotes = notes

	// Build list of packages.
	var packages []*database.Package
	for _, response := range responses {
		for _, v := range response.Vulnerabilities {
			packages = append(packages, v.FixedIn...)
		}
	}

	// Insert packages into the database.
	log.Tracef("beginning insertion of %d packages for update", len(packages))
	t := time.Now()
	err := database.InsertPackages(packages)
	log.Tracef("inserting %d packages took %v", len(packages), time.Since(t))
	if err != nil {
		log.Errorf("an error occured when inserting packages for update: %s", err)
		updateHealth(false)
		return
	}
	packages = nil

	// Build a list of vulnerabilties.
	var vulnerabilities []*database.Vulnerability
	for _, response := range responses {
		for _, v := range response.Vulnerabilities {
			var packageNodes []string
			for _, pkg := range v.FixedIn {
				packageNodes = append(packageNodes, pkg.Node)
			}
			vulnerabilities = append(vulnerabilities, &database.Vulnerability{ID: v.ID, Link: v.Link, Priority: v.Priority, Description: v.Description, FixedInNodes: packageNodes})
		}
	}
	responses = nil

	// Insert vulnerabilities into the database.
	log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
	t = time.Now()
	notifications, err := database.InsertVulnerabilities(vulnerabilities)
	log.Tracef("inserting %d vulnerabilities took %v", len(vulnerabilities), time.Since(t))
	if err != nil {
		log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
		updateHealth(false)
		return
	}
	vulnerabilities = nil

	// Insert notifications into the database.
	err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
	if err != nil {
		log.Errorf("an error occured when inserting notifications for update: %s", err)
		updateHealth(false)
		return
	}
	notifications = nil

	// Update flags in the database.
	for flagName, flagValue := range flags {
		database.UpdateFlag(flagName, flagValue)
	}

	// Update health depending on the status of the fetchers.
	updateHealth(status)
	if status {
		now := time.Now().UTC()
		database.UpdateFlag(flagName, strconv.FormatInt(now.Unix(), 10))
		healthLatestSuccessfulUpdate = now
	}
	log.Info("update finished")
}
Beispiel #6
0
// Run pops notifications from the database, lock them, send them, mark them as
// send and unlock them
//
// It uses an exponential backoff when POST requests fail
func (notifier *HTTPNotifier) Run(st *utils.Stopper) {
	defer st.End()

	whoAmI := uuid.New()
	log.Infof("HTTP notifier started. URL: %s. Lock Identifier: %s", notifier.url, whoAmI)

	for {
		node, notification, err := database.FindOneNotificationToSend(database.GetDefaultNotificationWrapper())
		if notification == nil || err != nil {
			if err != nil {
				log.Warningf("could not get notification to send: %s.", err)
			}

			if !st.Sleep(checkInterval) {
				break
			}

			continue
		}

		// Try to lock the notification
		hasLock, hasLockUntil := database.Lock(node, lockDuration, whoAmI)
		if !hasLock {
			continue
		}

		for backOff := time.Duration(0); ; backOff = timeutil.ExpBackoff(backOff, maxBackOff) {
			// Backoff, it happens when an error occurs during the communication
			// with the notification endpoint
			if backOff > 0 {
				// Renew lock before going to sleep if necessary
				if time.Now().Add(backOff).After(hasLockUntil.Add(-refreshLockAnticipation)) {
					hasLock, hasLockUntil = database.Lock(node, lockDuration, whoAmI)
					if !hasLock {
						log.Warning("lost lock ownership, aborting")
						break
					}
				}

				// Sleep
				if !st.Sleep(backOff) {
					return
				}
			}

			// Get notification content
			content, err := notification.GetContent()
			if err != nil {
				log.Warningf("could not get content of notification '%s': %s", notification.GetName(), err.Error())
				break
			}

			// Marshal the notification content
			jsonContent, err := json.Marshal(struct {
				Name, Type string
				Content    interface{}
			}{
				Name:    notification.GetName(),
				Type:    notification.GetType(),
				Content: content,
			})
			if err != nil {
				log.Errorf("could not marshal content of notification '%s': %s", notification.GetName(), err.Error())
				break
			}

			// Post notification
			req, _ := http.NewRequest("POST", notifier.url, bytes.NewBuffer(jsonContent))
			req.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			res, err := client.Do(req)
			if err != nil {
				log.Warningf("could not post notification '%s': %s", notification.GetName(), err.Error())
				continue
			}
			res.Body.Close()

			if res.StatusCode != 200 && res.StatusCode != 201 {
				log.Warningf("could not post notification '%s': got status code %d", notification.GetName(), res.StatusCode)
				continue
			}

			// Mark the notification as sent
			database.MarkNotificationAsSent(node)

			log.Infof("sent notification '%s' successfully", notification.GetName())
			break
		}

		if hasLock {
			database.Unlock(node, whoAmI)
		}
	}

	log.Info("HTTP notifier stopped")
}