func toFeatureVersions(criteria criteria) []database.FeatureVersion {
// There are duplicates in Red Hat .xml files.
// This map is for deduplication.
featureVersionParameters := make(map[string]database.FeatureVersion)
possibilities := getPossibilities(criteria)
for _, criterions := range possibilities {
var (
featureVersion database.FeatureVersion
osVersion int
err error
)
// Attempt to parse package data from trees of criterions.
for _, c := range criterions {
if strings.Contains(c.Comment, " is installed") {
const prefixLen = len("Red Hat Enterprise Linux ")
osVersion, err = strconv.Atoi(strings.TrimSpace(c.Comment[prefixLen : prefixLen+strings.Index(c.Comment[prefixLen:], " ")]))
if err != nil {
log.Warningf("could not parse Red Hat release version from: '%s'.", c.Comment)
}
} else if strings.Contains(c.Comment, " is earlier than ") {
const prefixLen = len(" is earlier than ")
featureVersion.Feature.Name = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")])
featureVersion.Version, err = types.NewVersion(c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:], err.Error())
}
}
}
if osVersion >= firstConsideredRHEL {
// TODO(vbatts) this is where features need multiple labels ('centos' and 'rhel')
featureVersion.Feature.Namespace.Name = "centos" + ":" + strconv.Itoa(osVersion)
} else {
continue
}
if featureVersion.Feature.Namespace.Name != "" && featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion
} else {
log.Warningf("could not determine a valid package from criterions: %v", criterions)
}
}
// Convert the map to slice.
var featureVersionParametersArray []database.FeatureVersion
for _, fv := range featureVersionParameters {
featureVersionParametersArray = append(featureVersionParametersArray, fv)
}
return featureVersionParametersArray
}
// Parse criterions into an array of FeatureVersion for storing into the database
func (f *OvalFetcher) ToFeatureVersions(possibilities [][]criterion) []database.FeatureVersion {
featureVersionParameters := make(map[string]database.FeatureVersion)
for _, criterions := range possibilities {
var (
featureVersion database.FeatureVersion
osVersion string
)
for _, c := range criterions {
if osVersion != "" && featureVersion.Feature.Name != "" &&
featureVersion.Version.String() != "" {
break
}
tmp_v := f.OsInfo.ParseOsVersion(c.Comment)
if tmp_v != "" {
osVersion = tmp_v
continue
}
tmp_p_name, tmp_p_version := f.OsInfo.ParsePackageNameVersion(c.Comment)
if tmp_p_version != "" && tmp_p_name != "" {
featureVersion.Feature.Name = tmp_p_name
featureVersion.Version, _ = types.NewVersion(tmp_p_version)
continue
}
log.Warningf("could not parse criteria: '%s'.", c.Comment)
}
if osVersion == "" {
log.Warning("No OS version found for criterions")
log.Warning(criterions)
continue
}
featureVersion.Feature.Namespace.Name = fmt.Sprintf("%s:%s", f.OsInfo.Namespace(), osVersion)
if featureVersion.Feature.Name != "" && featureVersion.Version.String() != "" {
featureVersionParameters[featureVersion.Feature.Namespace.Name+":"+featureVersion.Feature.Name] = featureVersion
} else {
log.Warningf("could not determine a valid package from criterions: %v", criterions)
}
}
var featureVersionParametersArray []database.FeatureVersion
for _, fv := range featureVersionParameters {
featureVersionParametersArray = append(featureVersionParametersArray, fv)
}
return featureVersionParametersArray
}
func toPackages(criteria criteria) []*database.Package {
// There are duplicates in Red Hat .xml files.
// This map is for deduplication.
packagesParameters := make(map[string]*database.Package)
possibilities := getPossibilities(criteria)
for _, criterions := range possibilities {
var (
pkg database.Package
osVersion int
err error
)
// Attempt to parse package data from trees of criterions.
for _, c := range criterions {
if strings.Contains(c.Comment, " is installed") {
const prefixLen = len("Red Hat Enterprise Linux ")
osVersion, err = strconv.Atoi(strings.TrimSpace(c.Comment[prefixLen : prefixLen+strings.Index(c.Comment[prefixLen:], " ")]))
if err != nil {
log.Warningf("could not parse Red Hat release version from: '%s'.", c.Comment)
}
} else if strings.Contains(c.Comment, " is earlier than ") {
const prefixLen = len(" is earlier than ")
pkg.Name = strings.TrimSpace(c.Comment[:strings.Index(c.Comment, " is earlier than ")])
pkg.Version, err = types.NewVersion(c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", c.Comment[strings.Index(c.Comment, " is earlier than ")+prefixLen:], err.Error())
}
}
}
if osVersion > firstConsideredRHEL {
pkg.OS = "centos" + ":" + strconv.Itoa(osVersion)
} else {
continue
}
if pkg.OS != "" && pkg.Name != "" && pkg.Version.String() != "" {
packagesParameters[pkg.Key()] = &pkg
} else {
log.Warningf("could not determine a valid package from criterions: %v", criterions)
}
}
// Convert the map to slice.
var packagesParametersArray []*database.Package
for _, p := range packagesParameters {
packagesParametersArray = append(packagesParametersArray, p)
}
return packagesParametersArray
}
func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
file, exists := data["lib/apk/db/installed"]
if !exists {
return []database.FeatureVersion{}, nil
}
// Iterate over each line in the "installed" file attempting to parse each
// package into a feature that will be stored in a set to guarantee
// uniqueness.
pkgSet := make(map[string]database.FeatureVersion)
ipkg := database.FeatureVersion{}
scanner := bufio.NewScanner(bytes.NewBuffer(file))
for scanner.Scan() {
line := scanner.Text()
if len(line) < 2 {
continue
}
// Parse the package name or version.
switch {
case line[:2] == "P:":
ipkg.Feature.Name = line[2:]
case line[:2] == "V:":
var err error
ipkg.Version, err = types.NewVersion(line[2:])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[2:], err.Error())
}
}
// If we have a whole feature, store it in the set and try to parse a new
// one.
if ipkg.Feature.Name != "" && ipkg.Version.String() != "" {
pkgSet[ipkg.Feature.Name+"#"+ipkg.Version.String()] = ipkg
ipkg = database.FeatureVersion{}
}
}
// Convert the map into a slice.
pkgs := make([]database.FeatureVersion, 0, len(pkgSet))
for _, pkg := range pkgSet {
pkgs = append(pkgs, pkg)
}
return pkgs, nil
}
// toPackages converts a path leading to one or multiple packages to Package structs, selecting the specified fields
func toPackages(path *path.Path, selectedFields []string) ([]*Package, error) {
var packages []*Package
var err error
saveFields(path, selectedFields, []string{FieldPackagePreviousVersion})
it, _ := path.BuildIterator().Optimize()
defer it.Close()
for cayley.RawNext(it) {
tags := make(map[string]graph.Value)
it.TagResults(tags)
pkg := Package{Node: store.NameOf(it.Result())}
for _, selectedField := range selectedFields {
switch selectedField {
case FieldPackageOS:
pkg.OS = store.NameOf(tags[FieldPackageOS])
case FieldPackageName:
pkg.Name = store.NameOf(tags[FieldPackageName])
case FieldPackageVersion:
pkg.Version, err = types.NewVersion(store.NameOf(tags[FieldPackageVersion]))
if err != nil {
log.Warningf("could not parse version of package %s: %s", pkg.Node, err.Error())
}
case FieldPackageNextVersion:
pkg.NextVersionNode = store.NameOf(tags[FieldPackageNextVersion])
case FieldPackagePreviousVersion:
pkg.PreviousVersionNode, err = toValue(cayley.StartPath(store, pkg.Node).In(FieldPackageNextVersion))
if err != nil {
log.Warningf("could not get previousVersion on package %s: %s.", pkg.Node, err.Error())
return []*Package{}, ErrInconsistent
}
default:
panic("unknown selectedField")
}
}
packages = append(packages, &pkg)
}
if it.Err() != nil {
log.Errorf("failed query in toPackages: %s", it.Err())
return []*Package{}, ErrBackendException
}
return packages, nil
}
func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
var rBytes []byte
rBytes, err = ioutil.ReadAll(r)
if err != nil {
return
}
var file secdb34File
err = yaml.Unmarshal(rBytes, &file)
if err != nil {
return
}
for _, pack := range file.Packages {
pkg := pack.Pkg
for versionStr, vulnStrs := range pkg.Fixes {
version, err := types.NewVersion(versionStr)
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error())
continue
}
for _, vulnStr := range vulnStrs {
var vuln database.Vulnerability
vuln.Severity = types.Unknown
vuln.Name = vulnStr
vuln.Link = nvdURLPrefix + vulnStr
vuln.FixedIn = []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "alpine:" + file.Distro},
Name: pkg.Name,
},
Version: version,
},
}
vulns = append(vulns, vuln)
}
}
}
return
}
func (f Feature) DatabaseModel() (database.FeatureVersion, error) {
var version types.Version
if f.Version == "None" {
version = types.MaxVersion
} else {
var err error
version, err = types.NewVersion(f.Version)
if err != nil {
return database.FeatureVersion{}, err
}
}
return database.FeatureVersion{
Feature: database.Feature{
Name: f.Name,
Namespace: database.Namespace{Name: f.Namespace},
},
Version: version,
}, nil
}
func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
var rBytes []byte
rBytes, err = ioutil.ReadAll(r)
if err != nil {
return
}
var file secdb33File
err = yaml.Unmarshal(rBytes, &file)
if err != nil {
return
}
for _, pack := range file.Packages {
pkg := pack.Pkg
for _, fix := range pkg.Fixes {
version, err := types.NewVersion(pkg.Version)
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
continue
}
vulns = append(vulns, database.Vulnerability{
Name: fix,
Severity: types.Unknown,
Link: nvdURLPrefix + fix,
FixedIn: []database.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "alpine:" + file.Distro},
Name: pkg.Name,
},
Version: version,
},
},
})
}
}
return
}
func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) {
mvulnerabilities := make(map[string]*database.Vulnerability)
unknownReleases = make(map[string]struct{})
for pkgName, pkgNode := range *data {
for vulnName, vulnNode := range pkgNode {
for releaseName, releaseNode := range vulnNode.Releases {
// Attempt to detect the release number.
if _, isReleaseKnown := database.DebianReleasesMapping[releaseName]; !isReleaseKnown {
unknownReleases[releaseName] = struct{}{}
continue
}
// Skip if the release is not affected.
if releaseNode.FixedVersion == "0" || releaseNode.Status == "undetermined" {
continue
}
// Get or create the vulnerability.
vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
if !vulnerabilityAlreadyExists {
vulnerability = &database.Vulnerability{
ID: vulnName,
Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""),
Priority: types.Unknown,
Description: vulnNode.Description,
}
}
// Set the priority of the vulnerability.
// In the JSON, a vulnerability has one urgency per package it affects.
// The highest urgency should be the one set.
urgency := urgencyToPriority(releaseNode.Urgency)
if urgency.Compare(vulnerability.Priority) > 0 {
vulnerability.Priority = urgency
}
// Determine the version of the package the vulnerability affects.
var version types.Version
var err error
if releaseNode.Status == "open" {
// Open means that the package is currently vulnerable in the latest
// version of this Debian release.
version = types.MaxVersion
} else if releaseNode.Status == "resolved" {
// Resolved means that the vulnerability has been fixed in
// "fixed_version" (if affected).
version, err = types.NewVersion(releaseNode.FixedVersion)
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", releaseNode.FixedVersion, err.Error())
continue
}
}
// Create and add the package.
pkg := &database.Package{
OS: "debian:" + database.DebianReleasesMapping[releaseName],
Name: pkgName,
Version: version,
}
vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode())
packages = append(packages, pkg)
// Store the vulnerability.
mvulnerabilities[vulnName] = vulnerability
}
}
}
// Convert the vulnerabilities map to a slice
for _, v := range mvulnerabilities {
vulnerabilities = append(vulnerabilities, v)
}
return
}
func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) {
unknownReleases = make(map[string]struct{})
readingDescription := false
scanner := bufio.NewScanner(fileContent)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// Skip any comments.
if strings.HasPrefix(line, "#") {
continue
}
// Parse the name.
if strings.HasPrefix(line, "Candidate:") {
vulnerability.Name = strings.TrimSpace(strings.TrimPrefix(line, "Candidate:"))
vulnerability.Link = fmt.Sprintf(cveURL, vulnerability.Name)
continue
}
// Parse the priority.
if strings.HasPrefix(line, "Priority:") {
priority := strings.TrimSpace(strings.TrimPrefix(line, "Priority:"))
// Handle syntax error: Priority: medium (heap-protector)
if strings.Contains(priority, " ") {
priority = priority[:strings.Index(priority, " ")]
}
vulnerability.Severity = ubuntuPriorityToSeverity(priority)
continue
}
// Parse the description.
if strings.HasPrefix(line, "Description:") {
readingDescription = true
vulnerability.Description = strings.TrimSpace(strings.TrimPrefix(line, "Description:")) // In case there is a formatting error and the description starts on the same line
continue
}
if readingDescription {
if strings.HasPrefix(line, "Ubuntu-Description:") || strings.HasPrefix(line, "Notes:") || strings.HasPrefix(line, "Bugs:") || strings.HasPrefix(line, "Priority:") || strings.HasPrefix(line, "Discovered-by:") || strings.HasPrefix(line, "Assigned-to:") {
readingDescription = false
} else {
vulnerability.Description = vulnerability.Description + " " + line
continue
}
}
// Try to parse the package that the vulnerability affects.
affectsCaptureArr := affectsCaptureRegexp.FindAllStringSubmatch(line, -1)
if len(affectsCaptureArr) > 0 {
affectsCapture := affectsCaptureArr[0]
md := map[string]string{}
for i, n := range affectsCapture {
md[affectsCaptureRegexpNames[i]] = strings.TrimSpace(n)
}
// Ignore Linux kernels.
if strings.HasPrefix(md["package"], "linux") {
continue
}
// Only consider the package if its status is needed, active, deferred, not-affected or
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" {
if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
continue
}
if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
unknownReleases[md["release"]] = struct{}{}
continue
}
var version types.Version
if md["status"] == "released" {
if md["note"] != "" {
var err error
version, err = types.NewVersion(md["note"])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", md["note"], err)
}
}
} else if md["status"] == "not-affected" {
version = types.MinVersion
} else {
version = types.MaxVersion
}
if version.String() == "" {
continue
}
// Create and add the new package.
featureVersion := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]},
Name: md["package"],
},
Version: version,
}
vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
}
}
}
// Trim extra spaces in the description
vulnerability.Description = strings.TrimSpace(vulnerability.Description)
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
if vulnerability.Link == "" {
vulnerability.Link = trackerURI
}
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
if vulnerability.Severity == "" {
vulnerability.Severity = types.Unknown
}
return
}
// Detect detects packages using var/lib/dpkg/status from the input data
func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/dpkg/status"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
var pkg database.FeatureVersion
var err error
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
// Package line
// Defines the name of the package
pkg.Feature.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = types.Version{}
} else if strings.HasPrefix(line, "Source: ") {
// Source line (Optionnal)
// Gives the name of the source package
// May also specifies a version
srcCapture := dpkgSrcCaptureRegexp.FindAllStringSubmatch(line, -1)[0]
md := map[string]string{}
for i, n := range srcCapture {
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}
pkg.Feature.Name = md["name"]
if md["version"] != "" {
pkg.Version, err = types.NewVersion(md["version"])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
}
}
} else if strings.HasPrefix(line, "Version: ") && pkg.Version.String() == "" {
// Version line
// Defines the version of the package
// This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter
pkg.Version, err = types.NewVersion(strings.TrimPrefix(line, "Version: "))
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
}
}
// Add the package to the result array if we have all the informations
if pkg.Feature.Name != "" && pkg.Version.String() != "" {
packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg
pkg.Feature.Name = ""
pkg.Version = types.Version{}
}
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}
return packages, nil
}
// Detect detects packages using var/lib/rpm/Packages from the input data
func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/rpm/Packages"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
// Write the required "Packages" file to disk
tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm")
defer os.RemoveAll(tmpDir)
if err != nil {
log.Errorf("could not create temporary folder for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
}
err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700)
if err != nil {
log.Errorf("could not create temporary file for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
}
// Query RPM
// We actually extract binary package names instead of source package names here because RHSA refers to package names
// In the dpkg system, we extract the source instead
out, err := utils.Exec(tmpDir, "rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n")
if err != nil {
log.Errorf("could not query RPM: %s. output: %s", err, string(out))
// Do not bubble up because we probably won't be able to fix it,
// the database must be corrupted
return []database.FeatureVersion{}, nil
}
scanner := bufio.NewScanner(strings.NewReader(string(out)))
for scanner.Scan() {
line := strings.Split(scanner.Text(), " ")
if len(line) != 2 {
// We may see warnings on some RPM versions:
// "warning: Generating 12 missing index(es), please wait..."
continue
}
// Ignore gpg-pubkey packages which are fake packages used to store GPG keys - they are not versionned properly.
if line[0] == "gpg-pubkey" {
continue
}
// Parse version
version, err := types.NewVersion(strings.Replace(line[1], "(none):", "", -1))
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
continue
}
// Add package
pkg := database.FeatureVersion{
Feature: database.Feature{
Name: line[0],
},
Version: version,
}
packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}
return packages, nil
}
