func toFeatureVersions(criteria criteria) []database.FeatureVersion { // There are duplicates in Red Hat .xml files. // This map is for deduplication. featureVersionParameters := make(map[string]database.FeatureVersion) possibilities := getPossibilities(criteria) for _, criterions := range possibilities { var ( featureVersion database.FeatureVersion osVersion int err error ) // Attempt to parse package data from trees of criterions. for _, c := range criterions { if strings.Contains(c.Comment, " is installed") { const prefixLen = len("Red Hat Enterprise Linux ") osVersion, err = strconv.Atoi(strings.TrimSpace(c.Comment[prefixLen : prefixLen+strings.Index(c.Comment[prefixLen:], " ")])) if err != nil { log.Warningf("could not parse Red Hat release version from: '%s'.", c.Comment) } } else if strings.Contains(c.Comment, " is earlier than ") { const prefixLen = len(" is earlier than ") featureVersion.Feature.Name = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")]) featureVersion.Version, err = types.NewVersion(c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:]) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:], err.Error()) } } } if osVersion >= firstConsideredRHEL { // TODO(vbatts) this is where features need multiple labels ('centos' and 'rhel') featureVersion.Feature.Namespace.Name = "centos" + ":" + strconv.Itoa(osVersion) } else { continue } if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion } else { log.Warningf("could not determine a valid package from criterions: %v", criterions) } } // Convert the map to slice. var featureVersionParametersArray []database.FeatureVersion for _, fv := range featureVersionParameters { featureVersionParametersArray = append(featureVersionParametersArray, fv) } return featureVersionParametersArray }
// Parse criterions into an array of FeatureVersion for storing into the database func (f *OvalFetcher) ToFeatureVersions(possibilities [][]criterion) []database.FeatureVersion { featureVersionParameters := make(map[string]database.FeatureVersion) for _, criterions := range possibilities { var ( featureVersion database.FeatureVersion osVersion string ) for _, c := range criterions { if osVersion != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { break } tmp_v := f.OsInfo.ParseOsVersion(c.Comment) if tmp_v != "" { osVersion = tmp_v continue } tmp_p_name, tmp_p_version := f.OsInfo.ParsePackageNameVersion(c.Comment) if tmp_p_version != "" && tmp_p_name != "" { featureVersion.Feature.Name = tmp_p_name featureVersion.Version, _ = types.NewVersion(tmp_p_version) continue } log.Warningf("could not parse criteria: '%s'.", c.Comment) } if osVersion == "" { log.Warning("No OS version found for criterions") log.Warning(criterions) continue } featureVersion.Feature.Namespace.Name = fmt.Sprintf("%s:%s", f.OsInfo.Namespace(), osVersion) if featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" { featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion } else { log.Warningf("could not determine a valid package from criterions: %v", criterions) } } var featureVersionParametersArray []database.FeatureVersion for _, fv := range featureVersionParameters { featureVersionParametersArray = append(featureVersionParametersArray, fv) } return featureVersionParametersArray }
func toPackages(criteria criteria) []*database.Package { // There are duplicates in Red Hat .xml files. // This map is for deduplication. packagesParameters := make(map[string]*database.Package) possibilities := getPossibilities(criteria) for _, criterions := range possibilities { var ( pkg database.Package osVersion int err error ) // Attempt to parse package data from trees of criterions. for _, c := range criterions { if strings.Contains(c.Comment, " is installed") { const prefixLen = len("Red Hat Enterprise Linux ") osVersion, err = strconv.Atoi(strings.TrimSpace(c.Comment[prefixLen : prefixLen+strings.Index(c.Comment[prefixLen:], " ")])) if err != nil { log.Warningf("could not parse Red Hat release version from: '%s'.", c.Comment) } } else if strings.Contains(c.Comment, " is earlier than ") { const prefixLen = len(" is earlier than ") pkg.Name = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")]) pkg.Version, err = types.NewVersion(c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:]) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:], err.Error()) } } } if osVersion > firstConsideredRHEL { pkg.OS = "centos" + ":" + strconv.Itoa(osVersion) } else { continue } if pkg.OS != "" && pkg.Name != "" && pkg.Version.String() != "" { packagesParameters[pkg.Key()] = &pkg } else { log.Warningf("could not determine a valid package from criterions: %v", criterions) } } // Convert the map to slice. var packagesParametersArray []*database.Package for _, p := range packagesParameters { packagesParametersArray = append(packagesParametersArray, p) } return packagesParametersArray }
func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { file, exists := data["lib/apk/db/installed"] if !exists { return []database.FeatureVersion{}, nil } // Iterate over each line in the "installed" file attempting to parse each // package into a feature that will be stored in a set to guarantee // uniqueness. pkgSet := make(map[string]database.FeatureVersion) ipkg := database.FeatureVersion{} scanner := bufio.NewScanner(bytes.NewBuffer(file)) for scanner.Scan() { line := scanner.Text() if len(line) < 2 { continue } // Parse the package name or version. switch { case line[:2] == "P:": ipkg.Feature.Name = line[2:] case line[:2] == "V:": var err error ipkg.Version, err = types.NewVersion(line[2:]) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", line[2:], err.Error()) } } // If we have a whole feature, store it in the set and try to parse a new // one. if ipkg.Feature.Name != "" && ipkg.Version.String() != "" { pkgSet[ipkg.Feature.Name+"#"+ipkg.Version.String()] = ipkg ipkg = database.FeatureVersion{} } } // Convert the map into a slice. pkgs := make([]database.FeatureVersion, 0, len(pkgSet)) for _, pkg := range pkgSet { pkgs = append(pkgs, pkg) } return pkgs, nil }
// toPackages converts a path leading to one or multiple packages to Package structs, selecting the specified fields func toPackages(path *path.Path, selectedFields []string) ([]*Package, error) { var packages []*Package var err error saveFields(path, selectedFields, []string{FieldPackagePreviousVersion}) it, _ := path.BuildIterator().Optimize() defer it.Close() for cayley.RawNext(it) { tags := make(map[string]graph.Value) it.TagResults(tags) pkg := Package{Node: store.NameOf(it.Result())} for _, selectedField := range selectedFields { switch selectedField { case FieldPackageOS: pkg.OS = store.NameOf(tags[FieldPackageOS]) case FieldPackageName: pkg.Name = store.NameOf(tags[FieldPackageName]) case FieldPackageVersion: pkg.Version, err = types.NewVersion(store.NameOf(tags[FieldPackageVersion])) if err != nil { log.Warningf("could not parse version of package %s: %s", pkg.Node, err.Error()) } case FieldPackageNextVersion: pkg.NextVersionNode = store.NameOf(tags[FieldPackageNextVersion]) case FieldPackagePreviousVersion: pkg.PreviousVersionNode, err = toValue(cayley.StartPath(store, pkg.Node).In(FieldPackageNextVersion)) if err != nil { log.Warningf("could not get previousVersion on package %s: %s.", pkg.Node, err.Error()) return []*Package{}, ErrInconsistent } default: panic("unknown selectedField") } } packages = append(packages, &pkg) } if it.Err() != nil { log.Errorf("failed query in toPackages: %s", it.Err()) return []*Package{}, ErrBackendException } return packages, nil }
func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { var rBytes []byte rBytes, err = ioutil.ReadAll(r) if err != nil { return } var file secdb34File err = yaml.Unmarshal(rBytes, &file) if err != nil { return } for _, pack := range file.Packages { pkg := pack.Pkg for versionStr, vulnStrs := range pkg.Fixes { version, err := types.NewVersion(versionStr) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error()) continue } for _, vulnStr := range vulnStrs { var vuln database.Vulnerability vuln.Severity = types.Unknown vuln.Name = vulnStr vuln.Link = nvdURLPrefix + vulnStr vuln.FixedIn = []database.FeatureVersion{ { Feature: database.Feature{ Namespace: database.Namespace{Name: "alpine:" + file.Distro}, Name: pkg.Name, }, Version: version, }, } vulns = append(vulns, vuln) } } } return }
func (f Feature) DatabaseModel() (database.FeatureVersion, error) { var version types.Version if f.Version == "None" { version = types.MaxVersion } else { var err error version, err = types.NewVersion(f.Version) if err != nil { return database.FeatureVersion{}, err } } return database.FeatureVersion{ Feature: database.Feature{ Name: f.Name, Namespace: database.Namespace{Name: f.Namespace}, }, Version: version, }, nil }
func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { var rBytes []byte rBytes, err = ioutil.ReadAll(r) if err != nil { return } var file secdb33File err = yaml.Unmarshal(rBytes, &file) if err != nil { return } for _, pack := range file.Packages { pkg := pack.Pkg for _, fix := range pkg.Fixes { version, err := types.NewVersion(pkg.Version) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error()) continue } vulns = append(vulns, database.Vulnerability{ Name: fix, Severity: types.Unknown, Link: nvdURLPrefix + fix, FixedIn: []database.FeatureVersion{ { Feature: database.Feature{ Namespace: database.Namespace{Name: "alpine:" + file.Distro}, Name: pkg.Name, }, Version: version, }, }, }) } } return }
func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) { mvulnerabilities := make(map[string]*database.Vulnerability) unknownReleases = make(map[string]struct{}) for pkgName, pkgNode := range *data { for vulnName, vulnNode := range pkgNode { for releaseName, releaseNode := range vulnNode.Releases { // Attempt to detect the release number. if _, isReleaseKnown := database.DebianReleasesMapping[releaseName]; !isReleaseKnown { unknownReleases[releaseName] = struct{}{} continue } // Skip if the release is not affected. if releaseNode.FixedVersion == "0" || releaseNode.Status == "undetermined" { continue } // Get or create the vulnerability. vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName] if !vulnerabilityAlreadyExists { vulnerability = &database.Vulnerability{ ID: vulnName, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), Priority: types.Unknown, Description: vulnNode.Description, } } // Set the priority of the vulnerability. // In the JSON, a vulnerability has one urgency per package it affects. // The highest urgency should be the one set. urgency := urgencyToPriority(releaseNode.Urgency) if urgency.Compare(vulnerability.Priority) > 0 { vulnerability.Priority = urgency } // Determine the version of the package the vulnerability affects. var version types.Version var err error if releaseNode.Status == "open" { // Open means that the package is currently vulnerable in the latest // version of this Debian release. version = types.MaxVersion } else if releaseNode.Status == "resolved" { // Resolved means that the vulnerability has been fixed in // "fixed_version" (if affected). version, err = types.NewVersion(releaseNode.FixedVersion) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", releaseNode.FixedVersion, err.Error()) continue } } // Create and add the package. pkg := &database.Package{ OS: "debian:" + database.DebianReleasesMapping[releaseName], Name: pkgName, Version: version, } vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode()) packages = append(packages, pkg) // Store the vulnerability. mvulnerabilities[vulnName] = vulnerability } } } // Convert the vulnerabilities map to a slice for _, v := range mvulnerabilities { vulnerabilities = append(vulnerabilities, v) } return }
func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) { unknownReleases = make(map[string]struct{}) readingDescription := false scanner := bufio.NewScanner(fileContent) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) // Skip any comments. if strings.HasPrefix(line, "#") { continue } // Parse the name. if strings.HasPrefix(line, "Candidate:") { vulnerability.Name = strings.TrimSpace(strings.TrimPrefix(line, "Candidate:")) vulnerability.Link = fmt.Sprintf(cveURL, vulnerability.Name) continue } // Parse the priority. if strings.HasPrefix(line, "Priority:") { priority := strings.TrimSpace(strings.TrimPrefix(line, "Priority:")) // Handle syntax error: Priority: medium (heap-protector) if strings.Contains(priority, " ") { priority = priority[:strings.Index(priority, " ")] } vulnerability.Severity = ubuntuPriorityToSeverity(priority) continue } // Parse the description. if strings.HasPrefix(line, "Description:") { readingDescription = true vulnerability.Description = strings.TrimSpace(strings.TrimPrefix(line, "Description:")) // In case there is a formatting error and the description starts on the same line continue } if readingDescription { if strings.HasPrefix(line, "Ubuntu-Description:") || strings.HasPrefix(line, "Notes:") || strings.HasPrefix(line, "Bugs:") || strings.HasPrefix(line, "Priority:") || strings.HasPrefix(line, "Discovered-by:") || strings.HasPrefix(line, "Assigned-to:") { readingDescription = false } else { vulnerability.Description = vulnerability.Description + " " + line continue } } // Try to parse the package that the vulnerability affects. affectsCaptureArr := affectsCaptureRegexp.FindAllStringSubmatch(line, -1) if len(affectsCaptureArr) > 0 { affectsCapture := affectsCaptureArr[0] md := map[string]string{} for i, n := range affectsCapture { md[affectsCaptureRegexpNames[i]] = strings.TrimSpace(n) } // Ignore Linux kernels. if strings.HasPrefix(md["package"], "linux") { continue } // Only consider the package if its status is needed, active, deferred, not-affected or // released. Ignore DNE (package does not exist), needs-triage, ignored, pending. if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" { if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored { continue } if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown { unknownReleases[md["release"]] = struct{}{} continue } var version types.Version if md["status"] == "released" { if md["note"] != "" { var err error version, err = types.NewVersion(md["note"]) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", md["note"], err) } } } else if md["status"] == "not-affected" { version = types.MinVersion } else { version = types.MaxVersion } if version.String() == "" { continue } // Create and add the new package. featureVersion := database.FeatureVersion{ Feature: database.Feature{ Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]}, Name: md["package"], }, Version: version, } vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion) } } } // Trim extra spaces in the description vulnerability.Description = strings.TrimSpace(vulnerability.Description) // If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker if vulnerability.Link == "" { vulnerability.Link = trackerURI } // If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown if vulnerability.Severity == "" { vulnerability.Severity = types.Unknown } return }
// Detect detects packages using var/lib/dpkg/status from the input data func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { f, hasFile := data["var/lib/dpkg/status"] if !hasFile { return []database.FeatureVersion{}, nil } // Create a map to store packages and ensure their uniqueness packagesMap := make(map[string]database.FeatureVersion) var pkg database.FeatureVersion var err error scanner := bufio.NewScanner(strings.NewReader(string(f))) for scanner.Scan() { line := scanner.Text() if strings.HasPrefix(line, "Package: ") { // Package line // Defines the name of the package pkg.Feature.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: ")) pkg.Version = types.Version{} } else if strings.HasPrefix(line, "Source: ") { // Source line (Optionnal) // Gives the name of the source package // May also specifies a version srcCapture := dpkgSrcCaptureRegexp.FindAllStringSubmatch(line, -1)[0] md := map[string]string{} for i, n := range srcCapture { md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n) } pkg.Feature.Name = md["name"] if md["version"] != "" { pkg.Version, err = types.NewVersion(md["version"]) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error()) } } } else if strings.HasPrefix(line, "Version: ") && pkg.Version.String() == "" { // Version line // Defines the version of the package // This version is less important than a version retrieved from a Source line // because the Debian vulnerabilities often skips the epoch from the Version field // which is not present in the Source version, and because +bX revisions don't matter pkg.Version, err = types.NewVersion(strings.TrimPrefix(line, "Version: ")) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error()) } } // Add the package to the result array if we have all the informations if pkg.Feature.Name != "" && pkg.Version.String() != "" { packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg pkg.Feature.Name = "" pkg.Version = types.Version{} } } // Convert the map to a slice packages := make([]database.FeatureVersion, 0, len(packagesMap)) for _, pkg := range packagesMap { packages = append(packages, pkg) } return packages, nil }
// Detect detects packages using var/lib/rpm/Packages from the input data func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { f, hasFile := data["var/lib/rpm/Packages"] if !hasFile { return []database.FeatureVersion{}, nil } // Create a map to store packages and ensure their uniqueness packagesMap := make(map[string]database.FeatureVersion) // Write the required "Packages" file to disk tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm") defer os.RemoveAll(tmpDir) if err != nil { log.Errorf("could not create temporary folder for RPM detection: %s", err) return []database.FeatureVersion{}, cerrors.ErrFilesystem } err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700) if err != nil { log.Errorf("could not create temporary file for RPM detection: %s", err) return []database.FeatureVersion{}, cerrors.ErrFilesystem } // Query RPM // We actually extract binary package names instead of source package names here because RHSA refers to package names // In the dpkg system, we extract the source instead out, err := utils.Exec(tmpDir, "rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n") if err != nil { log.Errorf("could not query RPM: %s. output: %s", err, string(out)) // Do not bubble up because we probably won't be able to fix it, // the database must be corrupted return []database.FeatureVersion{}, nil } scanner := bufio.NewScanner(strings.NewReader(string(out))) for scanner.Scan() { line := strings.Split(scanner.Text(), " ") if len(line) != 2 { // We may see warnings on some RPM versions: // "warning: Generating 12 missing index(es), please wait..." continue } // Ignore gpg-pubkey packages which are fake packages used to store GPG keys - they are not versionned properly. if line[0] == "gpg-pubkey" { continue } // Parse version version, err := types.NewVersion(strings.Replace(line[1], "(none):", "", -1)) if err != nil { log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error()) continue } // Add package pkg := database.FeatureVersion{ Feature: database.Feature{ Name: line[0], }, Version: version, } packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg } // Convert the map to a slice packages := make([]database.FeatureVersion, 0, len(packagesMap)) for _, pkg := range packagesMap { packages = append(packages, pkg) } return packages, nil }