// Update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. func Update() { log.Info("updating vulnerabilities") // Fetch updates. status, responses := fetch() // Merge responses. vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses) if err != nil { log.Errorf("an error occured when merging update responses: %s", err) return } responses = nil // TODO(Quentin-M): Complete informations using NVD // Insert packages. log.Tracef("beginning insertion of %d packages for update", len(packages)) err = database.InsertPackages(packages) if err != nil { log.Errorf("an error occured when inserting packages for update: %s", err) return } packages = nil // Insert vulnerabilities. log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) notifications, err := database.InsertVulnerabilities(vulnerabilities) if err != nil { log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) return } vulnerabilities = nil // Insert notifications into the database. err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) if err != nil { log.Errorf("an error occured when inserting notifications for update: %s", err) return } notifications = nil // Update flags and notes. for flagName, flagValue := range flags { database.UpdateFlag(flagName, flagValue) } database.UpdateFlag(notesFlagName, notes) // Update last successful update if every fetchers worked properly. if status { database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) } log.Info("update finished") }
// POSTVulnerabilities manually inserts a vulnerability into the database if it // does not exist yet. func POSTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) { var parameters *database.AbstractVulnerability if s, err := jsonhttp.ParseBody(r, ¶meters); err != nil { jsonhttp.RenderError(w, s, err) return } // Ensure that the vulnerability does not exist. vulnerability, err := database.FindOneVulnerability(parameters.ID, []string{}) if err != nil && err != cerrors.ErrNotFound { jsonhttp.RenderError(w, 0, err) return } if vulnerability != nil { jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("vulnerability already exists")) return } // Insert packages. packages := database.AbstractPackagesToPackages(parameters.AffectedPackages) err = database.InsertPackages(packages) if err != nil { jsonhttp.RenderError(w, 0, err) return } var pkgNodes []string for _, p := range packages { pkgNodes = append(pkgNodes, p.Node) } // Insert vulnerability. notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)}) if err != nil { jsonhttp.RenderError(w, 0, err) return } // Insert notifications. err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) if err != nil { jsonhttp.RenderError(w, 0, err) return } jsonhttp.Render(w, http.StatusCreated, nil) }
// PUTVulnerabilities updates a vulnerability if it exists. func PUTVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) { var parameters *database.AbstractVulnerability if s, err := httputils.ParseHTTPBody(r, ¶meters); err != nil { httputils.WriteHTTPError(w, s, err) return } parameters.ID = p.ByName("id") // Ensure that the vulnerability exists. _, err := database.FindOneVulnerability(parameters.ID, []string{}) if err != nil { httputils.WriteHTTPError(w, 0, err) return } // Insert packages. packages := database.AbstractPackagesToPackages(parameters.AffectedPackages) err = database.InsertPackages(packages) if err != nil { httputils.WriteHTTPError(w, 0, err) return } var pkgNodes []string for _, p := range packages { pkgNodes = append(pkgNodes, p.Node) } // Insert vulnerability. notifications, err := database.InsertVulnerabilities([]*database.Vulnerability{parameters.ToVulnerability(pkgNodes)}) if err != nil { httputils.WriteHTTPError(w, 0, err) return } // Insert notifications. err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) if err != nil { httputils.WriteHTTPError(w, 0, err) return } httputils.WriteHTTP(w, http.StatusCreated, nil) }
// Update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. func Update() { log.Info("updating vulnerabilities") // Fetch updates in parallel. var status = true var responseC = make(chan *FetcherResponse, 0) for n, f := range fetchers { go func(name string, fetcher Fetcher) { response, err := fetcher.FetchUpdate() if err != nil { log.Errorf("an error occured when fetching update '%s': %s.", name, err) status = false responseC <- nil return } responseC <- &response }(n, f) } // Collect results of updates. var responses []*FetcherResponse var notes []string for i := 0; i < len(fetchers); { select { case resp := <-responseC: if resp != nil { responses = append(responses, resp) notes = append(notes, resp.Notes...) } i++ } } close(responseC) // TODO(Quentin-M): Merge responses together // TODO(Quentin-M): Complete informations using NVD // Store flags out of the response struct. flags := make(map[string]string) for _, response := range responses { if response.FlagName != "" && response.FlagValue != "" { flags[response.FlagName] = response.FlagValue } } // Update health notes. healthNotes = notes // Build list of packages. var packages []*database.Package for _, response := range responses { for _, v := range response.Vulnerabilities { packages = append(packages, v.FixedIn...) } } // Insert packages into the database. log.Tracef("beginning insertion of %d packages for update", len(packages)) t := time.Now() err := database.InsertPackages(packages) log.Tracef("inserting %d packages took %v", len(packages), time.Since(t)) if err != nil { log.Errorf("an error occured when inserting packages for update: %s", err) updateHealth(false) return } packages = nil // Build a list of vulnerabilties. var vulnerabilities []*database.Vulnerability for _, response := range responses { for _, v := range response.Vulnerabilities { var packageNodes []string for _, pkg := range v.FixedIn { packageNodes = append(packageNodes, pkg.Node) } vulnerabilities = append(vulnerabilities, &database.Vulnerability{ID: v.ID, Link: v.Link, Priority: v.Priority, Description: v.Description, FixedInNodes: packageNodes}) } } responses = nil // Insert vulnerabilities into the database. log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) t = time.Now() notifications, err := database.InsertVulnerabilities(vulnerabilities) log.Tracef("inserting %d vulnerabilities took %v", len(vulnerabilities), time.Since(t)) if err != nil { log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) updateHealth(false) return } vulnerabilities = nil // Insert notifications into the database. err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) if err != nil { log.Errorf("an error occured when inserting notifications for update: %s", err) updateHealth(false) return } notifications = nil // Update flags in the database. for flagName, flagValue := range flags { database.UpdateFlag(flagName, flagValue) } // Update health depending on the status of the fetchers. updateHealth(status) if status { now := time.Now().UTC() database.UpdateFlag(flagName, strconv.FormatInt(now.Unix(), 10)) healthLatestSuccessfulUpdate = now } log.Info("update finished") }
// detectAndInsertInstalledAndRemovedPackages finds the installed and removed // package nodes and inserts the installed packages into the database. func detectAndInsertInstalledAndRemovedPackages(detectedOS string, packageList []*database.Package, parent *database.Layer) (installedNodes, removedNodes []string, err error) { // Get the parent layer's packages. parentPackageNodes, err := parent.AllPackages() if err != nil { return nil, nil, err } parentPackages, err := database.FindAllPackagesByNodes(parentPackageNodes, []string{database.FieldPackageName, database.FieldPackageVersion}) if err != nil { return nil, nil, err } // Map detected packages (name:version) string to packages. packagesNVMapToPackage := make(map[string]*database.Package) for _, p := range packageList { packagesNVMapToPackage[p.Name+":"+p.Version.String()] = p } // Map parent's packages (name:version) string to nodes. parentPackagesNVMapToNodes := make(map[string]string) for _, p := range parentPackages { parentPackagesNVMapToNodes[p.Name+":"+p.Version.String()] = p.Node } // Build a list of the parent layer's packages' node values. var parentPackagesNV []string for _, p := range parentPackages { parentPackagesNV = append(parentPackagesNV, p.Name+":"+p.Version.String()) } // Build a list of the layer packages' node values. var layerPackagesNV []string for _, p := range packageList { layerPackagesNV = append(layerPackagesNV, p.Name+":"+p.Version.String()) } // Calculate the installed and removed packages. removedPackagesNV := utils.CompareStringLists(parentPackagesNV, layerPackagesNV) installedPackagesNV := utils.CompareStringLists(layerPackagesNV, parentPackagesNV) // Build a list of all the installed packages. var installedPackages []*database.Package for _, nv := range installedPackagesNV { p, _ := packagesNVMapToPackage[nv] p.OS = detectedOS installedPackages = append(installedPackages, p) } // Insert that list into the database. err = database.InsertPackages(installedPackages) if err != nil { return nil, nil, err } // Build the list of installed package nodes. for _, p := range installedPackages { if p.Node != "" { installedNodes = append(installedNodes, p.Node) } } // Build the list of removed package nodes. for _, nv := range removedPackagesNV { node, _ := parentPackagesNVMapToNodes[nv] removedNodes = append(removedNodes, node) } return }
// detectContent downloads a layer's archive, extracts info from it and returns // an updated Layer struct. // // If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be // has been selectioned. func detectContent(ID, path string, parent *database.Layer) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) { data, err := getLayerData(path) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", ID, path, err) return } OS, err = detectOS(data, parent) if err != nil { return } log.Debugf("layer %s: OS is %s.", ID, OS) packageList, err := detectors.DetectPackages(data) if err != nil { log.Errorf("layer %s: package list could not be determined: %s", ID, err) return } // If there are any packages, that layer modified the package list. if len(packageList) > 0 { // It is possible that the OS could not be detected, in the case of a // first layer setting MAINTAINER only for instance. However, if the OS // is unknown and packages are detected, we have to return an error. if OS == "" { log.Errorf("layer %s: OS is unknown but %d packages have been detected", ID, len(packageList)) err = ErrUnsupported return } // If the layer has no parent, it can only add packages, not remove them. if parent == nil { // Build a list of the layer packages' node values. var installedPackages []*database.Package for _, p := range packageList { p.OS = OS installedPackages = append(installedPackages, p) } // Insert that list into the database. err = database.InsertPackages(installedPackages) if err != nil { return } // Set the InstalledPackageNodes field on content. for _, p := range installedPackages { if p.Node != "" { installedPackagesNodes = append(installedPackagesNodes, p.Node) } } } else { installedPackagesNodes, removedPackagesNodes, err = detectAndInsertInstalledAndRemovedPackages(OS, packageList, parent) if err != nil { return } } } log.Debugf("layer %s: detected %d packages: installs %d and removes %d packages", ID, len(packageList), len(installedPackagesNodes), len(removedPackagesNodes)) return }