func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error { // add and del are the FeatureVersion diff we should insert. var add []database.FeatureVersion var del []database.FeatureVersion if layer.Parent == nil { // There is no parent, every Features are added. add = append(add, layer.Features...) } else if layer.Parent != nil { // There is a parent, we need to diff the Features with it. // Build name:version structures. layerFeaturesMapNV, layerFeaturesNV := createNV(layer.Features) parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features) // Calculate the added and deleted FeatureVersions name:version. addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV) delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV) // Fill the structures containing the added and deleted FeatureVersions for _, nv := range addNV { add = append(add, *layerFeaturesMapNV[nv]) } for _, nv := range delNV { del = append(del, *parentLayerFeaturesMapNV[nv]) } } // Insert FeatureVersions in the database. addIDs, err := pgSQL.insertFeatureVersions(add) if err != nil { return err } delIDs, err := pgSQL.insertFeatureVersions(del) if err != nil { return err } // Insert diff in the database. if len(addIDs) > 0 { _, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "add", buildInputArray(addIDs)) if err != nil { return handleError("insertLayerDiffFeatureVersion.Add", err) } } if len(delIDs) > 0 { _, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "del", buildInputArray(delIDs)) if err != nil { return handleError("insertLayerDiffFeatureVersion.Del", err) } } return nil }
func layerEqual(expected, actual *Layer) bool { eq := true eq = eq && expected.Node == actual.Node eq = eq && expected.ID == actual.ID eq = eq && expected.ParentNode == actual.ParentNode eq = eq && expected.OS == actual.OS eq = eq && expected.EngineVersion == actual.EngineVersion eq = eq && len(utils.CompareStringLists(actual.SuccessorsNodes, expected.SuccessorsNodes)) == 0 && len(utils.CompareStringLists(expected.SuccessorsNodes, actual.SuccessorsNodes)) == 0 eq = eq && len(utils.CompareStringLists(actual.RemovedPackagesNodes, expected.RemovedPackagesNodes)) == 0 && len(utils.CompareStringLists(expected.RemovedPackagesNodes, actual.RemovedPackagesNodes)) == 0 eq = eq && len(utils.CompareStringLists(actual.InstalledPackagesNodes, expected.InstalledPackagesNodes)) == 0 && len(utils.CompareStringLists(expected.InstalledPackagesNodes, actual.InstalledPackagesNodes)) == 0 return eq }
// AllPackages computes the full list of packages that l has and return them as // nodes. // It requires that FieldLayerParent, FieldLayerContentInstalledPackages, // FieldLayerContentRemovedPackages fields has been selected on l func (l *Layer) AllPackages() ([]string, error) { var allPackages []string parent, err := l.Parent([]string{FieldLayerParent, FieldLayerPackages}) if err != nil { return []string{}, err } if parent != nil { allPackages, err = parent.AllPackages() if err != nil { return []string{}, err } } return append(utils.CompareStringLists(allPackages, l.RemovedPackagesNodes), l.InstalledPackagesNodes...), nil }
// applyFixedInDiff applies a FeatureVersion diff on a FeatureVersion list and returns the result. func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.FeatureVersion, bool) { currentMap, currentNames := createFeatureVersionNameMap(currentList) diffMap, diffNames := createFeatureVersionNameMap(diff) addedNames := utils.CompareStringLists(diffNames, currentNames) inBothNames := utils.CompareStringListsInBoth(diffNames, currentNames) different := false for _, name := range addedNames { if diffMap[name].Version == types.MinVersion { // MinVersion only makes sense when a Feature is already fixed in some version, // in which case we would be in the "inBothNames". continue } currentMap[name] = diffMap[name] different = true } for _, name := range inBothNames { fv := diffMap[name] if fv.Version == types.MinVersion { // MinVersion means that the Feature doesn't affect the Vulnerability anymore. delete(currentMap, name) different = true } else if fv.Version != currentMap[name].Version { // The version got updated. currentMap[name] = diffMap[name] different = true } } // Convert currentMap to a slice and return it. var newList []database.FeatureVersion for _, fv := range currentMap { newList = append(newList, fv) } return newList, different }
func (n *VulnerabilityPackageChangedNotification) GetContent() (interface{}, error) { // Returns the removed and added packages as well as the layers that // introduced the vulnerability in the past but don't anymore because of the // removed packages and the layers that now introduce the vulnerability // because of the added packages // Find vulnerability. vulnerability, err := FindOneVulnerability(n.VulnerabilityID, []string{FieldVulnerabilityID, FieldVulnerabilityLink, FieldVulnerabilityPriority, FieldVulnerabilityDescription, FieldVulnerabilityFixedIn}) if err != nil { return []byte{}, err } abstractVulnerability, err := vulnerability.ToAbstractVulnerability() if err != nil { return []byte{}, err } // First part of the answer : added/removed packages addedPackages, err := FindAllPackagesByNodes(n.AddedFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion, FieldPackagePreviousVersion}) if err != nil { return []byte{}, err } removedPackages, err := FindAllPackagesByNodes(n.RemovedFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion, FieldPackagePreviousVersion}) if err != nil { return []byte{}, err } // Second part of the answer var addedPackagesPreviousVersions []string for _, pkg := range addedPackages { previousVersions, err := pkg.PreviousVersions([]string{}) if err != nil { return []*Layer{}, err } for _, version := range previousVersions { addedPackagesPreviousVersions = append(addedPackagesPreviousVersions, version.Node) } } var removedPackagesPreviousVersions []string for _, pkg := range removedPackages { previousVersions, err := pkg.PreviousVersions([]string{}) if err != nil { return []*Layer{}, err } for _, version := range previousVersions { removedPackagesPreviousVersions = append(removedPackagesPreviousVersions, version.Node) } } newIntroducingLayers, err := FindAllLayersByAddedPackageNodes(addedPackagesPreviousVersions, []string{FieldLayerID}) if err != nil { return []byte{}, err } formerIntroducingLayers, err := FindAllLayersByAddedPackageNodes(removedPackagesPreviousVersions, []string{FieldLayerID}) if err != nil { return []byte{}, err } newIntroducingLayersIDs := []string{} // empty slice, not null for _, l := range newIntroducingLayers { newIntroducingLayersIDs = append(newIntroducingLayersIDs, l.ID) } formerIntroducingLayersIDs := []string{} // empty slice, not null for _, l := range formerIntroducingLayers { formerIntroducingLayersIDs = append(formerIntroducingLayersIDs, l.ID) } // Remove layers which appears both in new and former lists (eg. case of updated packages but still vulnerable) filteredNewIntroducingLayersIDs := utils.CompareStringLists(newIntroducingLayersIDs, formerIntroducingLayersIDs) filteredFormerIntroducingLayersIDs := utils.CompareStringLists(formerIntroducingLayersIDs, newIntroducingLayersIDs) return struct { Vulnerability *AbstractVulnerability AddedAffectedPackages, RemovedAffectedPackages []*AbstractPackage NewIntroducingLayersIDs, FormerIntroducingLayerIDs []string }{ Vulnerability: abstractVulnerability, AddedAffectedPackages: PackagesToAbstractPackages(addedPackages), RemovedAffectedPackages: PackagesToAbstractPackages(removedPackages), NewIntroducingLayersIDs: filteredNewIntroducingLayersIDs, FormerIntroducingLayerIDs: filteredFormerIntroducingLayersIDs, }, nil }
// InsertVulnerabilities inserts or updates several vulnerabilities in the database in one transaction // During an update, if the vulnerability was previously fixed by a version in a branch and a new package of that branch is specified, the previous one is deleted // Otherwise, it simply adds the defined packages, there is currently no way to delete affected packages. // // ID, Link, Priority and FixedInNodes fields have to be specified. Description is optionnal. func InsertVulnerabilities(vulnerabilities []*Vulnerability) ([]Notification, error) { if len(vulnerabilities) == 0 { return []Notification{}, nil } // Create required data structure var err error t := cayley.NewTransaction() cachedVulnerabilities := make(map[string]*Vulnerability) var notifications []Notification newVulnerabilityNotifications := make(map[string]*NewVulnerabilityNotification) vulnerabilityPriorityIncreasedNotifications := make(map[string]*VulnerabilityPriorityIncreasedNotification) vulnerabilityPackageChangedNotifications := make(map[string]*VulnerabilityPackageChangedNotification) // Iterate over all the vulnerabilities we need to insert/update for _, vulnerability := range vulnerabilities { // Check if the vulnerability already exists existingVulnerability, _ := cachedVulnerabilities[vulnerability.ID] if existingVulnerability == nil { existingVulnerability, err = FindOneVulnerability(vulnerability.ID, FieldVulnerabilityAll) if err != nil && err != cerrors.ErrNotFound { return []Notification{}, err } if existingVulnerability != nil { cachedVulnerabilities[vulnerability.ID] = existingVulnerability } } // Insert/Update vulnerability if existingVulnerability == nil { // The vulnerability does not exist, create it // Verify parameters if vulnerability.ID == "" || vulnerability.Link == "" || vulnerability.Priority == "" { log.Warningf("could not insert an incomplete vulnerability [ID: %s, Link: %s, Priority: %s]", vulnerability.ID, vulnerability.Link, vulnerability.Priority) return []Notification{}, cerrors.NewBadRequestError("Could not insert an incomplete vulnerability") } if !vulnerability.Priority.IsValid() { log.Warningf("could not insert a vulnerability which has an invalid priority [ID: %s, Link: %s, Priority: %s]. Valid priorities are: %v.", vulnerability.ID, vulnerability.Link, vulnerability.Priority, types.Priorities) return []Notification{}, cerrors.NewBadRequestError("Could not insert a vulnerability which has an invalid priority") } if len(vulnerability.FixedInNodes) == 0 { log.Warningf("could not insert a vulnerability which doesn't affect any package [ID: %s].", vulnerability.ID) return []Notification{}, cerrors.NewBadRequestError("could not insert a vulnerability which doesn't affect any package") } // Insert it vulnerability.Node = vulnerability.GetNode() t.AddQuad(cayley.Triple(vulnerability.Node, fieldIs, fieldVulnerabilityIsValue)) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityID, vulnerability.ID)) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityLink, vulnerability.Link)) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority))) t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description)) for _, p := range vulnerability.FixedInNodes { t.AddQuad(cayley.Triple(vulnerability.Node, FieldVulnerabilityFixedIn, p)) } // Add a notification notification := &NewVulnerabilityNotification{VulnerabilityID: vulnerability.ID} notifications = append(notifications, notification) newVulnerabilityNotifications[vulnerability.ID] = notification cachedVulnerabilities[vulnerability.ID] = vulnerability } else { // The vulnerability already exists, update it if vulnerability.Link != "" && existingVulnerability.Link != vulnerability.Link { t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, existingVulnerability.Link)) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityLink, vulnerability.Link)) existingVulnerability.Link = vulnerability.Link } if vulnerability.Priority != "" && vulnerability.Priority != types.Unknown && existingVulnerability.Priority != vulnerability.Priority { if !vulnerability.Priority.IsValid() { log.Warningf("could not update a vulnerability which has an invalid priority [ID: %s, Link: %s, Priority: %s]. Valid priorities are: %v.", vulnerability.ID, vulnerability.Link, vulnerability.Priority, types.Priorities) return []Notification{}, cerrors.NewBadRequestError("Could not update a vulnerability which has an invalid priority") } // Add a notification about the priority change if the new priority is higher and the vulnerability is not new if vulnerability.Priority.Compare(existingVulnerability.Priority) > 0 { if _, newVulnerabilityNotificationExists := newVulnerabilityNotifications[vulnerability.ID]; !newVulnerabilityNotificationExists { // Any priorityChangeNotification already ? if existingPriorityNotification, _ := vulnerabilityPriorityIncreasedNotifications[vulnerability.ID]; existingPriorityNotification != nil { // There is a priority change notification, replace it but keep the old priority field existingPriorityNotification.NewPriority = vulnerability.Priority } else { // No previous notification, just add a new one notification := &VulnerabilityPriorityIncreasedNotification{OldPriority: existingVulnerability.Priority, NewPriority: vulnerability.Priority, VulnerabilityID: existingVulnerability.ID} notifications = append(notifications, notification) vulnerabilityPriorityIncreasedNotifications[vulnerability.ID] = notification } } } t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(existingVulnerability.Priority))) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityPriority, string(vulnerability.Priority))) existingVulnerability.Priority = vulnerability.Priority } if vulnerability.Description != "" && existingVulnerability.Description != vulnerability.Description { t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, existingVulnerability.Description)) t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityDescription, vulnerability.Description)) existingVulnerability.Description = vulnerability.Description } newFixedInNodes := utils.CompareStringLists(vulnerability.FixedInNodes, existingVulnerability.FixedInNodes) if len(newFixedInNodes) > 0 { var removedNodes []string var addedNodes []string existingVulnerabilityFixedInPackages, err := FindAllPackagesByNodes(existingVulnerability.FixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion}) if err != nil { return []Notification{}, err } newFixedInPackages, err := FindAllPackagesByNodes(newFixedInNodes, []string{FieldPackageOS, FieldPackageName, FieldPackageVersion}) if err != nil { return []Notification{}, err } for _, p := range newFixedInPackages { for _, ep := range existingVulnerabilityFixedInPackages { if p.Branch() == ep.Branch() { // A link to this package branch already exist and is not the same version, we will delete it t.RemoveQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, ep.Node)) var index int for i, n := range existingVulnerability.FixedInNodes { if n == ep.Node { index = i break } } existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes[index:], existingVulnerability.FixedInNodes[index+1:]...) removedNodes = append(removedNodes, ep.Node) } } t.AddQuad(cayley.Triple(existingVulnerability.Node, FieldVulnerabilityFixedIn, p.Node)) existingVulnerability.FixedInNodes = append(existingVulnerability.FixedInNodes, p.Node) addedNodes = append(addedNodes, p.Node) } // Add notification about the FixedIn modification if the vulnerability is not new if _, newVulnerabilityNotificationExists := newVulnerabilityNotifications[vulnerability.ID]; !newVulnerabilityNotificationExists { // Any VulnerabilityPackageChangedNotification already ? if existingPackageNotification, _ := vulnerabilityPackageChangedNotifications[vulnerability.ID]; existingPackageNotification != nil { // There is a priority change notification, add the packages modifications to it existingPackageNotification.AddedFixedInNodes = append(existingPackageNotification.AddedFixedInNodes, addedNodes...) existingPackageNotification.RemovedFixedInNodes = append(existingPackageNotification.RemovedFixedInNodes, removedNodes...) } else { // No previous notification, just add a new one notification := &VulnerabilityPackageChangedNotification{VulnerabilityID: vulnerability.ID, AddedFixedInNodes: addedNodes, RemovedFixedInNodes: removedNodes} notifications = append(notifications, notification) vulnerabilityPackageChangedNotifications[vulnerability.ID] = notification } } } } } // Apply transaction if err = store.ApplyTransaction(t); err != nil { log.Errorf("failed transaction (InsertVulnerabilities): %s", err) return []Notification{}, ErrTransaction } return notifications, nil }
// detectAndInsertInstalledAndRemovedPackages finds the installed and removed // package nodes and inserts the installed packages into the database. func detectAndInsertInstalledAndRemovedPackages(detectedOS string, packageList []*database.Package, parent *database.Layer) (installedNodes, removedNodes []string, err error) { // Get the parent layer's packages. parentPackageNodes, err := parent.AllPackages() if err != nil { return nil, nil, err } parentPackages, err := database.FindAllPackagesByNodes(parentPackageNodes, []string{database.FieldPackageName, database.FieldPackageVersion}) if err != nil { return nil, nil, err } // Map detected packages (name:version) string to packages. packagesNVMapToPackage := make(map[string]*database.Package) for _, p := range packageList { packagesNVMapToPackage[p.Name+":"+p.Version.String()] = p } // Map parent's packages (name:version) string to nodes. parentPackagesNVMapToNodes := make(map[string]string) for _, p := range parentPackages { parentPackagesNVMapToNodes[p.Name+":"+p.Version.String()] = p.Node } // Build a list of the parent layer's packages' node values. var parentPackagesNV []string for _, p := range parentPackages { parentPackagesNV = append(parentPackagesNV, p.Name+":"+p.Version.String()) } // Build a list of the layer packages' node values. var layerPackagesNV []string for _, p := range packageList { layerPackagesNV = append(layerPackagesNV, p.Name+":"+p.Version.String()) } // Calculate the installed and removed packages. removedPackagesNV := utils.CompareStringLists(parentPackagesNV, layerPackagesNV) installedPackagesNV := utils.CompareStringLists(layerPackagesNV, parentPackagesNV) // Build a list of all the installed packages. var installedPackages []*database.Package for _, nv := range installedPackagesNV { p, _ := packagesNVMapToPackage[nv] p.OS = detectedOS installedPackages = append(installedPackages, p) } // Insert that list into the database. err = database.InsertPackages(installedPackages) if err != nil { return nil, nil, err } // Build the list of installed package nodes. for _, p := range installedPackages { if p.Node != "" { installedNodes = append(installedNodes, p.Node) } } // Build the list of removed package nodes. for _, nv := range removedPackagesNV { node, _ := parentPackagesNVMapToNodes[nv] removedNodes = append(removedNodes, node) } return }
func TestRaceAffects(t *testing.T) { datastore, err := openDatabaseForTest("RaceAffects", false) if err != nil { t.Error(err) return } defer datastore.Close() // Insert the Feature on which we'll work. feature := database.Feature{ Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"}, Name: "TestRaceAffecturesFeature1", } _, err = datastore.insertFeature(feature) if err != nil { t.Error(err) return } // Initialize random generator and enforce max procs. rand.Seed(time.Now().UnixNano()) runtime.GOMAXPROCS(runtime.NumCPU()) // Generate FeatureVersions. featureVersions := make([]database.FeatureVersion, numFeatureVersions) for i := 0; i < numFeatureVersions; i++ { version := rand.Intn(numFeatureVersions) featureVersions[i] = database.FeatureVersion{ Feature: feature, Version: types.NewVersionUnsafe(strconv.Itoa(version)), } } // Generate vulnerabilities. // They are mapped by fixed version, which will make verification really easy afterwards. vulnerabilities := make(map[int][]database.Vulnerability) for i := 0; i < numVulnerabilities; i++ { version := rand.Intn(numFeatureVersions) + 1 // if _, ok := vulnerabilities[version]; !ok { // vulnerabilities[version] = make([]database.Vulnerability) // } vulnerability := database.Vulnerability{ Name: uuid.New(), Namespace: feature.Namespace, FixedIn: []database.FeatureVersion{ { Feature: feature, Version: types.NewVersionUnsafe(strconv.Itoa(version)), }, }, Severity: types.Unknown, } vulnerabilities[version] = append(vulnerabilities[version], vulnerability) } // Insert featureversions and vulnerabilities in parallel. var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for _, vulnerabilitiesM := range vulnerabilities { for _, vulnerability := range vulnerabilitiesM { err = datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true) assert.Nil(t, err) } } fmt.Println("finished to insert vulnerabilities") }() go func() { defer wg.Done() for i := 0; i < len(featureVersions); i++ { featureVersions[i].ID, err = datastore.insertFeatureVersion(featureVersions[i]) assert.Nil(t, err) } fmt.Println("finished to insert featureVersions") }() wg.Wait() // Verify consistency now. var actualAffectedNames []string var expectedAffectedNames []string for _, featureVersion := range featureVersions { featureVersionVersion, _ := strconv.Atoi(featureVersion.Version.String()) // Get actual affects. rows, err := datastore.Query(searchComplexTestFeatureVersionAffects, featureVersion.ID) assert.Nil(t, err) defer rows.Close() var vulnName string for rows.Next() { err = rows.Scan(&vulnName) if !assert.Nil(t, err) { continue } actualAffectedNames = append(actualAffectedNames, vulnName) } if assert.Nil(t, rows.Err()) { rows.Close() } // Get expected affects. for i := numVulnerabilities; i > featureVersionVersion; i-- { for _, vulnerability := range vulnerabilities[i] { expectedAffectedNames = append(expectedAffectedNames, vulnerability.Name) } } assert.Len(t, utils.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0) assert.Len(t, utils.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0) } }