func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error {
// add and del are the FeatureVersion diff we should insert.
var add []database.FeatureVersion
var del []database.FeatureVersion
if layer.Parent == nil {
// There is no parent, every Features are added.
add = append(add, layer.Features...)
} else if layer.Parent != nil {
// There is a parent, we need to diff the Features with it.
// Build name:version structures.
layerFeaturesMapNV, layerFeaturesNV := createNV(layer.Features)
parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features)
// Calculate the added and deleted FeatureVersions name:version.
addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV)
delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV)
// Fill the structures containing the added and deleted FeatureVersions
for _, nv := range addNV {
add = append(add, *layerFeaturesMapNV[nv])
}
for _, nv := range delNV {
del = append(del, *parentLayerFeaturesMapNV[nv])
}
}
// Insert FeatureVersions in the database.
addIDs, err := pgSQL.insertFeatureVersions(add)
if err != nil {
return err
}
delIDs, err := pgSQL.insertFeatureVersions(del)
if err != nil {
return err
}
// Insert diff in the database.
if len(addIDs) > 0 {
_, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "add", buildInputArray(addIDs))
if err != nil {
return handleError("insertLayerDiffFeatureVersion.Add", err)
}
}
if len(delIDs) > 0 {
_, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "del", buildInputArray(delIDs))
if err != nil {
return handleError("insertLayerDiffFeatureVersion.Del", err)
}
}
return nil
}
func layerEqual(expected, actual *Layer) bool {
eq := true
eq = eq && expected.Node == actual.Node
eq = eq && expected.ID == actual.ID
eq = eq && expected.ParentNode == actual.ParentNode
eq = eq && expected.OS == actual.OS
eq = eq && expected.EngineVersion == actual.EngineVersion
eq = eq && len(utils.CompareStringLists(actual.SuccessorsNodes, expected.SuccessorsNodes)) == 0 && len(utils.CompareStringLists(expected.SuccessorsNodes, actual.SuccessorsNodes)) == 0
eq = eq && len(utils.CompareStringLists(actual.RemovedPackagesNodes, expected.RemovedPackagesNodes)) == 0 && len(utils.CompareStringLists(expected.RemovedPackagesNodes, actual.RemovedPackagesNodes)) == 0
eq = eq && len(utils.CompareStringLists(actual.InstalledPackagesNodes, expected.InstalledPackagesNodes)) == 0 && len(utils.CompareStringLists(expected.InstalledPackagesNodes, actual.InstalledPackagesNodes)) == 0
return eq
}
// AllPackages computes the full list of packages that l has and return them as
// nodes.
// It requires that FieldLayerParent, FieldLayerContentInstalledPackages,
// FieldLayerContentRemovedPackages fields has been selected on l
func (l *Layer) AllPackages() ([]string, error) {
var allPackages []string
parent, err := l.Parent([]string{FieldLayerParent, FieldLayerPackages})
if err != nil {
return []string{}, err
}
if parent != nil {
allPackages, err = parent.AllPackages()
if err != nil {
return []string{}, err
}
}
return append(utils.CompareStringLists(allPackages, l.RemovedPackagesNodes), l.InstalledPackagesNodes...), nil
}
// applyFixedInDiff applies a FeatureVersion diff on a FeatureVersion list and returns the result.
func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.FeatureVersion, bool) {
currentMap, currentNames := createFeatureVersionNameMap(currentList)
diffMap, diffNames := createFeatureVersionNameMap(diff)
addedNames := utils.CompareStringLists(diffNames, currentNames)
inBothNames := utils.CompareStringListsInBoth(diffNames, currentNames)
different := false
for _, name := range addedNames {
if diffMap[name].Version == types.MinVersion {
// MinVersion only makes sense when a Feature is already fixed in some version,
// in which case we would be in the "inBothNames".
continue
}
currentMap[name] = diffMap[name]
different = true
}
for _, name := range inBothNames {
fv := diffMap[name]
if fv.Version == types.MinVersion {
// MinVersion means that the Feature doesn't affect the Vulnerability anymore.
delete(currentMap, name)
different = true
} else if fv.Version != currentMap[name].Version {
// The version got updated.
currentMap[name] = diffMap[name]
different = true
}
}
// Convert currentMap to a slice and return it.
var newList []database.FeatureVersion
for _, fv := range currentMap {
newList = append(newList, fv)
}
return newList, different
}
func (n *VulnerabilityPackageChangedNotification) GetContent() (interface{}, error) {
// Returns the removed and added packages as well as the layers that
// introduced the vulnerability in the past but don't anymore because of the
// removed packages and the layers that now introduce the vulnerability
// because of the added packages
// Find vulnerability.
vulnerability, err := FindOneVulnerability(n.VulnerabilityID, []string{FieldVulnerabilityID, FieldVulnerabilityLink, FieldVulnerabilityPriority, FieldVulnerabilityDescription, FieldVulnerabilityFixedIn})
if err != nil {
return []byte{}, err
}
abstractVulnerability, err := vulnerability.ToAbstractVulnerability()
if err != nil {
return []byte{}, err
}
// First part of the answer : added/removed packages
addedPackages, err := FindAllPackagesByNodes(n.AddedFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion, FieldPackagePreviousVersion})
if err != nil {
return []byte{}, err
}
removedPackages, err := FindAllPackagesByNodes(n.RemovedFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion, FieldPackagePreviousVersion})
if err != nil {
return []byte{}, err
}
// Second part of the answer
var addedPackagesPreviousVersions []string
for _, pkg := range addedPackages {
previousVersions, err := pkg.PreviousVersions([]string{})
if err != nil {
return []*Layer{}, err
}
for _, version := range previousVersions {
addedPackagesPreviousVersions = append(addedPackagesPreviousVersions, version.Node)
}
}
var removedPackagesPreviousVersions []string
for _, pkg := range removedPackages {
previousVersions, err := pkg.PreviousVersions([]string{})
if err != nil {
return []*Layer{}, err
}
for _, version := range previousVersions {
removedPackagesPreviousVersions = append(removedPackagesPreviousVersions, version.Node)
}
}
newIntroducingLayers, err := FindAllLayersByAddedPackageNodes(addedPackagesPreviousVersions, []string{FieldLayerID})
if err != nil {
return []byte{}, err
}
formerIntroducingLayers, err := FindAllLayersByAddedPackageNodes(removedPackagesPreviousVersions, []string{FieldLayerID})
if err != nil {
return []byte{}, err
}
newIntroducingLayersIDs := []string{} // empty slice, not null
for _, l := range newIntroducingLayers {
newIntroducingLayersIDs = append(newIntroducingLayersIDs, l.ID)
}
formerIntroducingLayersIDs := []string{} // empty slice, not null
for _, l := range formerIntroducingLayers {
formerIntroducingLayersIDs = append(formerIntroducingLayersIDs, l.ID)
}
// Remove layers which appears both in new and former lists (eg. case of updated packages but still vulnerable)
filteredNewIntroducingLayersIDs := utils.CompareStringLists(newIntroducingLayersIDs, formerIntroducingLayersIDs)
filteredFormerIntroducingLayersIDs := utils.CompareStringLists(formerIntroducingLayersIDs, newIntroducingLayersIDs)
return struct {
Vulnerability *AbstractVulnerability
AddedAffectedPackages, RemovedAffectedPackages []*AbstractPackage
NewIntroducingLayersIDs, FormerIntroducingLayerIDs []string
}{
Vulnerability: abstractVulnerability,
AddedAffectedPackages: PackagesToAbstractPackages(addedPackages),
RemovedAffectedPackages: PackagesToAbstractPackages(removedPackages),
NewIntroducingLayersIDs: filteredNewIntroducingLayersIDs,
FormerIntroducingLayerIDs: filteredFormerIntroducingLayersIDs,
}, nil
}
// InsertVulnerabilities inserts or updates several vulnerabilities in the database in one transaction
// During an update, if the vulnerability was previously fixed by a version in a branch and a new package of that branch is specified, the previous one is deleted
// Otherwise, it simply adds the defined packages, there is currently no way to delete affected packages.
//
// ID, Link, Priority and FixedInNodes fields have to be specified. Description is optionnal.
func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, error) {
if len(vulnerabilities) == 0 {
return []Notification{}, nil
}
// Create required data structure
var err error
t := cayley.NewTransaction()
cachedVulnerabilities := make(map[string]*Vulnerability)
var notifications []Notification
newVulnerabilityNotifications := make(map[string]*NewVulnerabilityNotification)
vulnerabilityPriorityIncreasedNotifications := make(map[string]*VulnerabilityPriorityIncreasedNotification)
vulnerabilityPackageChangedNotifications := make(map[string]*VulnerabilityPackageChangedNotification)
// Iterate over all the vulnerabilities we need to insert/update
for _, vulnerability := range vulnerabilities {
// Check if the vulnerability already exists
existingVulnerability, _ := cachedVulnerabilities[vulnerability.ID]
if existingVulnerability == nil {
existingVulnerability, err = FindOneVulnerability(vulnerability.ID, FieldVulnerabilityAll)
if err != nil && err != cerrors.ErrNotFound {
return []Notification{}, err
}
if existingVulnerability != nil {
cachedVulnerabilities[vulnerability.ID] = existingVulnerability
}
}
// Insert/Update vulnerability
if existingVulnerability == nil {
// The vulnerability does not exist, create it
// Verify parameters
if vulnerability.ID == "" || vulnerability.Link == "" || vulnerability.Priority == "" {
log.Warningf("could not insert an incomplete vulnerability [ID: %s, Link: %s, Priority: %s]", vulnerability.ID, vulnerability.Link, vulnerability.Priority)
return []Notification{}, cerrors.NewBadRequestError("Could not insert an incomplete vulnerability")
}
if !vulnerability.Priority.IsValid() {
log.Warningf("could not insert a vulnerability which has an invalid priority [ID: %s, Link: %s, Priority: %s]. Valid priorities are: %v.", vulnerability.ID, vulnerability.Link, vulnerability.Priority, types.Priorities)
return []Notification{}, cerrors.NewBadRequestError("Could not insert a vulnerability which has an invalid priority")
}
if len(vulnerability.FixedInNodes) == 0 {
log.Warningf("could not insert a vulnerability which doesn't affect any package [ID: %s].", vulnerability.ID)
return []Notification{}, cerrors.NewBadRequestError("could not insert a vulnerability which doesn't affect any package")
}
// Insert it
vulnerability.Node = vulnerability.GetNode()
t.AddQuad(cayley.Triple(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue))
t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID))
t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link))
t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority)))
t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description))
for _, p := range vulnerability.FixedInNodes {
t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p))
}
// Add a notification
notification := &NewVulnerabilityNotification{VulnerabilityID: vulnerability.ID}
notifications = append(notifications, notification)
newVulnerabilityNotifications[vulnerability.ID] = notification
cachedVulnerabilities[vulnerability.ID] = vulnerability
} else {
// The vulnerability already exists, update it
if vulnerability.Link != "" && existingVulnerability.Link != vulnerability.Link {
t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link))
t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link))
existingVulnerability.Link = vulnerability.Link
}
if vulnerability.Priority != "" && vulnerability.Priority != types.Unknown && existingVulnerability.Priority != vulnerability.Priority {
if !vulnerability.Priority.IsValid() {
log.Warningf("could not update a vulnerability which has an invalid priority [ID: %s, Link: %s, Priority: %s]. Valid priorities are: %v.", vulnerability.ID, vulnerability.Link, vulnerability.Priority, types.Priorities)
return []Notification{}, cerrors.NewBadRequestError("Could not update a vulnerability which has an invalid priority")
}
// Add a notification about the priority change if the new priority is higher and the vulnerability is not new
if vulnerability.Priority.Compare(existingVulnerability.Priority) > 0 {
if _, newVulnerabilityNotificationExists := newVulnerabilityNotifications[vulnerability.ID]; !newVulnerabilityNotificationExists {
// Any priorityChangeNotification already ?
if existingPriorityNotification, _ := vulnerabilityPriorityIncreasedNotifications[vulnerability.ID]; existingPriorityNotification != nil {
// There is a priority change notification, replace it but keep the old priority field
existingPriorityNotification.NewPriority = vulnerability.Priority
} else {
// No previous notification, just add a new one
notification := &VulnerabilityPriorityIncreasedNotification{OldPriority: existingVulnerability.Priority, NewPriority: vulnerability.Priority, VulnerabilityID: existingVulnerability.ID}
notifications = append(notifications, notification)
vulnerabilityPriorityIncreasedNotifications[vulnerability.ID] = notification
}
}
}
t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority)))
t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority)))
existingVulnerability.Priority = vulnerability.Priority
}
if vulnerability.Description != "" && existingVulnerability.Description != vulnerability.Description {
t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description))
t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description))
existingVulnerability.Description = vulnerability.Description
}
newFixedInNodes := utils.CompareStringLists(vulnerability.FixedInNodes, existingVulnerability.FixedInNodes)
if len(newFixedInNodes) > 0 {
var removedNodes []string
var addedNodes []string
existingVulnerabilityFixedInPackages, err := FindAllPackagesByNodes(existingVulnerability.FixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion})
if err != nil {
return []Notification{}, err
}
newFixedInPackages, err := FindAllPackagesByNodes(newFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion})
if err != nil {
return []Notification{}, err
}
for _, p := range newFixedInPackages {
for _, ep := range existingVulnerabilityFixedInPackages {
if p.Branch() == ep.Branch() {
// A link to this package branch already exist and is not the same version, we will delete it
t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node))
var index int
for i, n := range existingVulnerability.FixedInNodes {
if n == ep.Node {
index = i
break
}
}
existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes[index:], existingVulnerability.FixedInNodes[index+1:]...)
removedNodes = append(removedNodes, ep.Node)
}
}
t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node))
existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes, p.Node)
addedNodes = append(addedNodes, p.Node)
}
// Add notification about the FixedIn modification if the vulnerability is not new
if _, newVulnerabilityNotificationExists := newVulnerabilityNotifications[vulnerability.ID]; !newVulnerabilityNotificationExists {
// Any VulnerabilityPackageChangedNotification already ?
if existingPackageNotification, _ := vulnerabilityPackageChangedNotifications[vulnerability.ID]; existingPackageNotification != nil {
// There is a priority change notification, add the packages modifications to it
existingPackageNotification.AddedFixedInNodes = append(existingPackageNotification.AddedFixedInNodes, addedNodes...)
existingPackageNotification.RemovedFixedInNodes = append(existingPackageNotification.RemovedFixedInNodes, removedNodes...)
} else {
// No previous notification, just add a new one
notification := &VulnerabilityPackageChangedNotification{VulnerabilityID: vulnerability.ID, AddedFixedInNodes: addedNodes, RemovedFixedInNodes: removedNodes}
notifications = append(notifications, notification)
vulnerabilityPackageChangedNotifications[vulnerability.ID] = notification
}
}
}
}
}
// Apply transaction
if err = store.ApplyTransaction(t); err != nil {
log.Errorf("failed transaction (InsertVulnerabilities): %s", err)
return []Notification{}, ErrTransaction
}
return notifications, nil
}
// detectAndInsertInstalledAndRemovedPackages finds the installed and removed
// package nodes and inserts the installed packages into the database.
func detectAndInsertInstalledAndRemovedPackages(detectedOS string, packageList []*database.Package, parent *database.Layer) (installedNodes, removedNodes []string, err error) {
// Get the parent layer's packages.
parentPackageNodes, err := parent.AllPackages()
if err != nil {
return nil, nil, err
}
parentPackages, err := database.FindAllPackagesByNodes(parentPackageNodes, []string{database.FieldPackageName, database.FieldPackageVersion})
if err != nil {
return nil, nil, err
}
// Map detected packages (name:version) string to packages.
packagesNVMapToPackage := make(map[string]*database.Package)
for _, p := range packageList {
packagesNVMapToPackage[p.Name+":"+p.Version.String()] = p
}
// Map parent's packages (name:version) string to nodes.
parentPackagesNVMapToNodes := make(map[string]string)
for _, p := range parentPackages {
parentPackagesNVMapToNodes[p.Name+":"+p.Version.String()] = p.Node
}
// Build a list of the parent layer's packages' node values.
var parentPackagesNV []string
for _, p := range parentPackages {
parentPackagesNV = append(parentPackagesNV, p.Name+":"+p.Version.String())
}
// Build a list of the layer packages' node values.
var layerPackagesNV []string
for _, p := range packageList {
layerPackagesNV = append(layerPackagesNV, p.Name+":"+p.Version.String())
}
// Calculate the installed and removed packages.
removedPackagesNV := utils.CompareStringLists(parentPackagesNV, layerPackagesNV)
installedPackagesNV := utils.CompareStringLists(layerPackagesNV, parentPackagesNV)
// Build a list of all the installed packages.
var installedPackages []*database.Package
for _, nv := range installedPackagesNV {
p, _ := packagesNVMapToPackage[nv]
p.OS = detectedOS
installedPackages = append(installedPackages, p)
}
// Insert that list into the database.
err = database.InsertPackages(installedPackages)
if err != nil {
return nil, nil, err
}
// Build the list of installed package nodes.
for _, p := range installedPackages {
if p.Node != "" {
installedNodes = append(installedNodes, p.Node)
}
}
// Build the list of removed package nodes.
for _, nv := range removedPackagesNV {
node, _ := parentPackagesNVMapToNodes[nv]
removedNodes = append(removedNodes, node)
}
return
}
func TestRaceAffects(t *testing.T) {
datastore, err := openDatabaseForTest("RaceAffects", false)
if err != nil {
t.Error(err)
return
}
defer datastore.Close()
// Insert the Feature on which we'll work.
feature := database.Feature{
Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"},
Name: "TestRaceAffecturesFeature1",
}
_, err = datastore.insertFeature(feature)
if err != nil {
t.Error(err)
return
}
// Initialize random generator and enforce max procs.
rand.Seed(time.Now().UnixNano())
runtime.GOMAXPROCS(runtime.NumCPU())
// Generate FeatureVersions.
featureVersions := make([]database.FeatureVersion, numFeatureVersions)
for i := 0; i < numFeatureVersions; i++ {
version := rand.Intn(numFeatureVersions)
featureVersions[i] = database.FeatureVersion{
Feature: feature,
Version: types.NewVersionUnsafe(strconv.Itoa(version)),
}
}
// Generate vulnerabilities.
// They are mapped by fixed version, which will make verification really easy afterwards.
vulnerabilities := make(map[int][]database.Vulnerability)
for i := 0; i < numVulnerabilities; i++ {
version := rand.Intn(numFeatureVersions) + 1
// if _, ok := vulnerabilities[version]; !ok {
// vulnerabilities[version] = make([]database.Vulnerability)
// }
vulnerability := database.Vulnerability{
Name: uuid.New(),
Namespace: feature.Namespace,
FixedIn: []database.FeatureVersion{
{
Feature: feature,
Version: types.NewVersionUnsafe(strconv.Itoa(version)),
},
},
Severity: types.Unknown,
}
vulnerabilities[version] = append(vulnerabilities[version], vulnerability)
}
// Insert featureversions and vulnerabilities in parallel.
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for _, vulnerabilitiesM := range vulnerabilities {
for _, vulnerability := range vulnerabilitiesM {
err = datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true)
assert.Nil(t, err)
}
}
fmt.Println("finished to insert vulnerabilities")
}()
go func() {
defer wg.Done()
for i := 0; i < len(featureVersions); i++ {
featureVersions[i].ID, err = datastore.insertFeatureVersion(featureVersions[i])
assert.Nil(t, err)
}
fmt.Println("finished to insert featureVersions")
}()
wg.Wait()
// Verify consistency now.
var actualAffectedNames []string
var expectedAffectedNames []string
for _, featureVersion := range featureVersions {
featureVersionVersion, _ := strconv.Atoi(featureVersion.Version.String())
// Get actual affects.
rows, err := datastore.Query(searchComplexTestFeatureVersionAffects,
featureVersion.ID)
assert.Nil(t, err)
defer rows.Close()
var vulnName string
for rows.Next() {
err = rows.Scan(&vulnName)
if !assert.Nil(t, err) {
continue
}
actualAffectedNames = append(actualAffectedNames, vulnName)
}
if assert.Nil(t, rows.Err()) {
rows.Close()
}
// Get expected affects.
for i := numVulnerabilities; i > featureVersionVersion; i-- {
for _, vulnerability := range vulnerabilities[i] {
expectedAffectedNames = append(expectedAffectedNames, vulnerability.Name)
}
}
assert.Len(t, utils.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0)
assert.Len(t, utils.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0)
}
}