func detectOS(data map[string][]byte, parent *database.Layer) (detectedOS string, err error) {
detectedOS = detectors.DetectOS(data)
// Attempt to detect the OS from the parent layer.
if detectedOS == "" && parent != nil {
detectedOS, err = parent.OperatingSystem()
if err != nil {
return "", err
}
}
// If the detectedOS is not in the supported OS list, the OS is unsupported.
if detectedOS != "" {
isSupported := false
for _, osPrefix := range SupportedOS {
if strings.HasPrefix(detectedOS, osPrefix) {
isSupported = true
break
}
}
if !isSupported {
return "", ErrUnsupported
}
}
return
}
// Process detects the OS of a layer, the packages it installs/removes, and
// then stores everything in the database.
func Process(ID, parentID, path string) error {
if ID == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have ID")
}
if path == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a path")
}
log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)
// Check to see if the layer is already in the database.
layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
if err != nil && err != cerrors.ErrNotFound {
return err
}
var parent *database.Layer
if layer != nil {
// The layer is already in the database, check if we need to update it.
if layer.EngineVersion >= Version {
log.Debugf("layer %s: layer content has already been processed in the past with engine %d. Current engine is %d. skipping analysis", ID, layer.EngineVersion, Version)
return nil
}
log.Debugf("layer %s: layer content has been analyzed in the past with engine %d. Current engine is %d. analyzing again", ID, layer.EngineVersion, Version)
} else {
// The layer is a new one, create a base struct that we will fill.
layer = &database.Layer{ID: ID, EngineVersion: Version}
// Check to make sure that the parent's layer has already been processed.
if parentID != "" {
parent, err = database.FindOneLayerByID(parentID, []string{database.FieldLayerOS, database.FieldLayerPackages, database.FieldLayerPackages})
if err != nil && err != cerrors.ErrNotFound {
return err
}
if parent == nil {
log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", ID, parentID)
return ErrParentUnknown
}
layer.ParentNode = parent.GetNode()
}
}
// Analyze the content.
layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
if err != nil {
return err
}
return database.InsertLayer(layer)
}
// detectAndInsertInstalledAndRemovedPackages finds the installed and removed
// package nodes and inserts the installed packages into the database.
func detectAndInsertInstalledAndRemovedPackages(detectedOS string, packageList []*database.Package, parent *database.Layer) (installedNodes, removedNodes []string, err error) {
// Get the parent layer's packages.
parentPackageNodes, err := parent.AllPackages()
if err != nil {
return nil, nil, err
}
parentPackages, err := database.FindAllPackagesByNodes(parentPackageNodes, []string{database.FieldPackageName, database.FieldPackageVersion})
if err != nil {
return nil, nil, err
}
// Map detected packages (name:version) string to packages.
packagesNVMapToPackage := make(map[string]*database.Package)
for _, p := range packageList {
packagesNVMapToPackage[p.Name+":"+p.Version.String()] = p
}
// Map parent's packages (name:version) string to nodes.
parentPackagesNVMapToNodes := make(map[string]string)
for _, p := range parentPackages {
parentPackagesNVMapToNodes[p.Name+":"+p.Version.String()] = p.Node
}
// Build a list of the parent layer's packages' node values.
var parentPackagesNV []string
for _, p := range parentPackages {
parentPackagesNV = append(parentPackagesNV, p.Name+":"+p.Version.String())
}
// Build a list of the layer packages' node values.
var layerPackagesNV []string
for _, p := range packageList {
layerPackagesNV = append(layerPackagesNV, p.Name+":"+p.Version.String())
}
// Calculate the installed and removed packages.
removedPackagesNV := utils.CompareStringLists(parentPackagesNV, layerPackagesNV)
installedPackagesNV := utils.CompareStringLists(layerPackagesNV, parentPackagesNV)
// Build a list of all the installed packages.
var installedPackages []*database.Package
for _, nv := range installedPackagesNV {
p, _ := packagesNVMapToPackage[nv]
p.OS = detectedOS
installedPackages = append(installedPackages, p)
}
// Insert that list into the database.
err = database.InsertPackages(installedPackages)
if err != nil {
return nil, nil, err
}
// Build the list of installed package nodes.
for _, p := range installedPackages {
if p.Node != "" {
installedNodes = append(installedNodes, p.Node)
}
}
// Build the list of removed package nodes.
for _, nv := range removedPackagesNV {
node, _ := parentPackagesNVMapToNodes[nv]
removedNodes = append(removedNodes, node)
}
return
}
func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
subquery := "all"
if withFeatures {
subquery += "/features"
} else if withVulnerabilities {
subquery += "/features+vulnerabilities"
}
defer observeQueryTime("FindLayer", subquery, time.Now())
// Find the layer
var layer database.Layer
var parentID zero.Int
var parentName zero.String
var namespaceID zero.Int
var namespaceName sql.NullString
t := time.Now()
err := pgSQL.QueryRow(searchLayer, name).
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID,
&namespaceName)
observeQueryTime("FindLayer", "searchLayer", t)
if err != nil {
return layer, handleError("searchLayer", err)
}
if !parentID.IsZero() {
layer.Parent = &database.Layer{
Model: database.Model{ID: int(parentID.Int64)},
Name: parentName.String,
}
}
if !namespaceID.IsZero() {
layer.Namespace = &database.Namespace{
Model: database.Model{ID: int(namespaceID.Int64)},
Name: namespaceName.String,
}
}
// Find its features
if withFeatures || withVulnerabilities {
// Create a transaction to disable hash/merge joins as our experiments have shown that
// PostgreSQL 9.4 makes bad planning decisions about:
// - joining the layer tree to feature versions and feature
// - joining the feature versions to affected/fixed feature version and vulnerabilities
// It would for instance do a merge join between affected feature versions (300 rows, estimated
// 3000 rows) and fixed in feature version (100k rows). In this case, it is much more
// preferred to use a nested loop.
tx, err := pgSQL.Begin()
if err != nil {
return layer, handleError("FindLayer.Begin()", err)
}
defer tx.Commit()
_, err = tx.Exec(disableHashJoin)
if err != nil {
log.Warningf("FindLayer: could not disable hash join: %s", err)
}
_, err = tx.Exec(disableMergeJoin)
if err != nil {
log.Warningf("FindLayer: could not disable merge join: %s", err)
}
t = time.Now()
featureVersions, err := getLayerFeatureVersions(tx, layer.ID)
observeQueryTime("FindLayer", "getLayerFeatureVersions", t)
if err != nil {
return layer, err
}
layer.Features = featureVersions
if withVulnerabilities {
// Load the vulnerabilities that affect the FeatureVersions.
t = time.Now()
err := loadAffectedBy(tx, layer.Features)
observeQueryTime("FindLayer", "loadAffectedBy", t)
if err != nil {
return layer, err
}
}
}
return layer, nil
}
// Internally, only Feature additions/removals are stored for each layer. If a layer has a parent,
// the Feature list will be compared to the parent's Feature list and the difference will be stored.
// Note that when the Namespace of a layer differs from its parent, it is expected that several
// Feature that were already included a parent will have their Namespace updated as well
// (happens when Feature detectors relies on the detected layer Namespace). However, if the listed
// Feature has the same Name/Version as its parent, InsertLayer considers that the Feature hasn't
// been modified.
func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
tf := time.Now()
// Verify parameters
if layer.Name == "" {
log.Warning("could not insert a layer which has an empty Name")
return cerrors.NewBadRequestError("could not insert a layer which has an empty Name")
}
// Get a potentially existing layer.
existingLayer, err := pgSQL.FindLayer(layer.Name, true, false)
if err != nil && err != cerrors.ErrNotFound {
return err
} else if err == nil {
if existingLayer.EngineVersion >= layer.EngineVersion {
// The layer exists and has an equal or higher engine version, do nothing.
return nil
}
layer.ID = existingLayer.ID
}
// We do `defer observeQueryTime` here because we don't want to observe existing layers.
defer observeQueryTime("InsertLayer", "all", tf)
// Get parent ID.
var parentID zero.Int
if layer.Parent != nil {
if layer.Parent.ID == 0 {
log.Warning("Parent is expected to be retrieved from database when inserting a layer.")
return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.")
}
parentID = zero.IntFrom(int64(layer.Parent.ID))
}
// Find or insert namespace if provided.
var namespaceID zero.Int
if layer.Namespace != nil {
n, err := pgSQL.insertNamespace(*layer.Namespace)
if err != nil {
return err
}
namespaceID = zero.IntFrom(int64(n))
} else if layer.Namespace == nil && layer.Parent != nil {
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
if layer.Parent.Namespace != nil {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
}
}
// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return handleError("InsertLayer.Begin()", err)
}
if layer.ID == 0 {
// Insert a new layer.
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID).
Scan(&layer.ID)
if err != nil {
tx.Rollback()
if isErrUniqueViolation(err) {
// Ignore this error, another process collided.
log.Debug("Attempted to insert duplicate layer.")
return nil
}
return handleError("insertLayer", err)
}
} else {
// Update an existing layer.
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID)
if err != nil {
tx.Rollback()
return handleError("updateLayer", err)
}
// Remove all existing Layer_diff_FeatureVersion.
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
if err != nil {
tx.Rollback()
return handleError("removeLayerDiffFeatureVersion", err)
}
}
// Update Layer_diff_FeatureVersion now.
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
if err != nil {
tx.Rollback()
return err
}
// Commit transaction.
err = tx.Commit()
if err != nil {
tx.Rollback()
return handleError("InsertLayer.Commit()", err)
}
return nil
}
func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
f7 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature7",
},
Version: types.NewVersionUnsafe("0.01"),
}
l3, _ := datastore.FindLayer("TestInsertLayer3", true, false)
l3u := database.Layer{
Name: l3.Name,
Parent: l3.Parent,
Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"},
Features: []database.FeatureVersion{f7},
}
l4u := database.Layer{
Name: "TestInsertLayer4",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Features: []database.FeatureVersion{f7},
EngineVersion: 2,
}
// Try to re-insert without increasing the EngineVersion.
err := datastore.InsertLayer(l3u)
assert.Nil(t, err)
l3uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name)
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
assert.Len(t, l3uf.Features, len(l3.Features))
}
// Update layer l3.
// Verify that the Namespace, EngineVersion and FeatureVersions got updated.
l3u.EngineVersion = 2
err = datastore.InsertLayer(l3u)
assert.Nil(t, err)
l3uf, err = datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name)
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
if assert.Len(t, l3uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l3uf.Features[0])
}
}
// Update layer l4.
// Verify that the Namespace got updated from its new Parent's, and also verify the
// EnginVersion and FeatureVersions.
l4u.Parent = &l3uf
err = datastore.InsertLayer(l4u)
assert.Nil(t, err)
l4uf, err := datastore.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name)
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
if assert.Len(t, l4uf.Features, 1) {
assert.True(t, cmpFV(l3uf.Features[0], f7), "Updated layer should have %#v but actually have %#v", f7, l4uf.Features[0])
}
}
}