// Process detects the OS of a layer, the packages it installs/removes, and
// then stores everything in the database.
func Process(ID, parentID, path string) error {
if ID == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have ID")
}
if path == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a path")
}
log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)
// Check to see if the layer is already in the database.
layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
if err != nil && err != cerrors.ErrNotFound {
return err
}
var parent *database.Layer
if layer != nil {
// The layer is already in the database, check if we need to update it.
if layer.EngineVersion >= Version {
log.Debugf("layer %s: layer content has already been processed in the past with engine %d. Current engine is %d. skipping analysis", ID, layer.EngineVersion, Version)
return nil
}
log.Debugf("layer %s: layer content has been analyzed in the past with engine %d. Current engine is %d. analyzing again", ID, layer.EngineVersion, Version)
} else {
// The layer is a new one, create a base struct that we will fill.
layer = &database.Layer{ID: ID, EngineVersion: Version}
// Check to make sure that the parent's layer has already been processed.
if parentID != "" {
parent, err = database.FindOneLayerByID(parentID, []string{database.FieldLayerOS, database.FieldLayerPackages, database.FieldLayerPackages})
if err != nil && err != cerrors.ErrNotFound {
return err
}
if parent == nil {
log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", ID, parentID)
return ErrParentUnknown
}
layer.ParentNode = parent.GetNode()
}
}
// Analyze the content.
layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
if err != nil {
return err
}
return database.InsertLayer(layer)
}
func TestDistUpgrade(t *testing.T) {
database.Open("memstore", "")
defer database.Close()
_, f, _, _ := runtime.Caller(0)
path := path.Join(path.Dir(f)) + "/testdata/DistUpgrade/"
// blank.tar: MAINTAINER Quentin MACHU <quentin.machu.fr>
// wheezy.tar: FROM debian:wheezy
// jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && apt-get -y dist-upgrade
assert.Nil(t, Process("blank", "", path+"blank.tar.gz"))
assert.Nil(t, Process("wheezy", "blank", path+"wheezy.tar.gz"))
assert.Nil(t, Process("jessie", "wheezy", path+"jessie.tar.gz"))
wheezy, err := database.FindOneLayerByID("wheezy", database.FieldLayerAll)
if assert.Nil(t, err) {
assert.Equal(t, "debian:7", wheezy.OS)
assert.Len(t, wheezy.InstalledPackagesNodes, 52)
assert.Len(t, wheezy.RemovedPackagesNodes, 0)
jessie, err := database.FindOneLayerByID("jessie", database.FieldLayerAll)
if assert.Nil(t, err) {
assert.Equal(t, "debian:8", jessie.OS)
assert.Len(t, jessie.InstalledPackagesNodes, 66)
assert.Len(t, jessie.RemovedPackagesNodes, 44)
packageNodes, err := jessie.AllPackages()
if assert.Nil(t, err) {
// These packages haven't been upgraded
nonUpgradedPackages := []database.Package{
database.Package{Name: "libtext-wrapi18n-perl", Version: types.NewVersionUnsafe("0.06-7")},
database.Package{Name: "libtext-charwidth-perl", Version: types.NewVersionUnsafe("0.04-7")},
database.Package{Name: "libtext-iconv-perl", Version: types.NewVersionUnsafe("1.7-5")},
database.Package{Name: "mawk", Version: types.NewVersionUnsafe("1.3.3-17")},
database.Package{Name: "insserv", Version: types.NewVersionUnsafe("1.14.0-5")},
database.Package{Name: "db", Version: types.NewVersionUnsafe("5.1.29-5")},
database.Package{Name: "ustr", Version: types.NewVersionUnsafe("1.0.4-3")},
database.Package{Name: "xz-utils", Version: types.NewVersionUnsafe("5.1.1alpha+20120614-2")},
}
for _, p := range nonUpgradedPackages {
p.OS = "debian:7"
assert.Contains(t, packageNodes, p.GetNode(), "Jessie layer doesn't have %s but it should.", p)
}
for _, p := range nonUpgradedPackages {
p.OS = "debian:8"
assert.NotContains(t, packageNodes, p.GetNode(), "Jessie layer has %s but it shouldn't.", p)
}
}
}
}
}
// GETLayersVulnerabilities returns the complete list of vulnerabilities that
// a layer has if it exists.
func GETLayersVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Get minumum priority parameter.
minimumPriority := types.Priority(r.URL.Query().Get("minimumPriority"))
if minimumPriority == "" {
minimumPriority = "High" // Set default priority to High
} else if !minimumPriority.IsValid() {
jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("invalid priority"))
return
}
// Find layer
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerParent, database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find layer's packages.
packagesNodes, err := layer.AllPackages()
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find vulnerabilities.
vulnerabilities, err := getVulnerabilitiesFromLayerPackagesNodes(packagesNodes, minimumPriority, []string{database.FieldVulnerabilityID, database.FieldVulnerabilityLink, database.FieldVulnerabilityPriority, database.FieldVulnerabilityDescription})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
jsonhttp.Render(w, http.StatusOK, struct{ Vulnerabilities []*database.Vulnerability }{Vulnerabilities: vulnerabilities})
}
// GETLayersPackagesDiff returns the list of packages that a layer installs and
// removes if it exists.
func GETLayersPackagesDiff(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Find layer.
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find layer's packages.
installedPackages, removedPackages := make([]*database.Package, 0), make([]*database.Package, 0)
if len(layer.InstalledPackagesNodes) > 0 {
installedPackages, err = database.FindAllPackagesByNodes(layer.InstalledPackagesNodes, []string{database.FieldPackageOS, database.FieldPackageName, database.FieldPackageVersion})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
}
if len(layer.RemovedPackagesNodes) > 0 {
removedPackages, err = database.FindAllPackagesByNodes(layer.RemovedPackagesNodes, []string{database.FieldPackageOS, database.FieldPackageName, database.FieldPackageVersion})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
}
jsonhttp.Render(w, http.StatusOK, struct{ InstalledPackages, RemovedPackages []*database.Package }{InstalledPackages: installedPackages, RemovedPackages: removedPackages})
}
// GETLayersPackages returns the complete list of packages that a layer has
// if it exists.
func GETLayersPackages(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Find layer
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerParent, database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find layer's packages.
packagesNodes, err := layer.AllPackages()
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
packages := []*database.Package{}
if len(packagesNodes) > 0 {
packages, err = database.FindAllPackagesByNodes(packagesNodes, []string{database.FieldPackageOS, database.FieldPackageName, database.FieldPackageVersion})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
}
jsonhttp.Render(w, http.StatusOK, struct{ Packages []*database.Package }{Packages: packages})
}
// POSTBatchLayersVulnerabilities returns the complete list of vulnerabilities
// that the provided layers have, if they all exist.
func POSTBatchLayersVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Parse body
var parameters POSTBatchLayersVulnerabilitiesParameters
if s, err := jsonhttp.ParseBody(r, ¶meters); err != nil {
jsonhttp.RenderError(w, s, err)
return
}
if len(parameters.LayersIDs) == 0 {
jsonhttp.RenderError(w, http.StatusBadRequest, errors.New("at least one LayerID query parameter must be provided"))
return
}
// Get minumum priority parameter.
minimumPriority := types.Priority(r.URL.Query().Get("minimumPriority"))
if minimumPriority == "" {
minimumPriority = "High" // Set default priority to High
} else if !minimumPriority.IsValid() {
jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("invalid priority"))
return
}
response := make(map[string]interface{})
// For each LayerID parameter
for _, layerID := range parameters.LayersIDs {
// Find layer
layer, err := database.FindOneLayerByID(layerID, []string{database.FieldLayerParent, database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find layer's packages.
packagesNodes, err := layer.AllPackages()
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find vulnerabilities.
vulnerabilities, err := getVulnerabilitiesFromLayerPackagesNodes(packagesNodes, minimumPriority, []string{database.FieldVulnerabilityID, database.FieldVulnerabilityLink, database.FieldVulnerabilityPriority, database.FieldVulnerabilityDescription})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
response[layerID] = struct{ Vulnerabilities []*database.Vulnerability }{Vulnerabilities: vulnerabilities}
}
jsonhttp.Render(w, http.StatusOK, response)
}
// GETLayersOS returns the operating system of a layer if it exists.
// It uses not only the specified layer but also its parent layers if necessary.
// An empty OS string is returned if no OS has been detected.
func GETLayersOS(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Find layer.
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerParent, database.FieldLayerOS})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Get OS.
os, err := layer.OperatingSystem()
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
jsonhttp.Render(w, http.StatusOK, struct{ OS string }{OS: os})
}
// GETLayersVulnerabilitiesDiff returns the list of vulnerabilities that a layer
// adds and removes if it exists.
func GETLayersVulnerabilitiesDiff(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Get minumum priority parameter.
minimumPriority := types.Priority(r.URL.Query().Get("minimumPriority"))
if minimumPriority == "" {
minimumPriority = "High" // Set default priority to High
} else if !minimumPriority.IsValid() {
jsonhttp.RenderError(w, 0, cerrors.NewBadRequestError("invalid priority"))
return
}
// Find layer.
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Selected fields for vulnerabilities.
selectedFields := []string{database.FieldVulnerabilityID, database.FieldVulnerabilityLink, database.FieldVulnerabilityPriority, database.FieldVulnerabilityDescription}
// Find vulnerabilities for installed packages.
addedVulnerabilities, err := getVulnerabilitiesFromLayerPackagesNodes(layer.InstalledPackagesNodes, minimumPriority, selectedFields)
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find vulnerabilities for removed packages.
removedVulnerabilities, err := getVulnerabilitiesFromLayerPackagesNodes(layer.RemovedPackagesNodes, minimumPriority, selectedFields)
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Remove vulnerabilities which appears both in added and removed lists (eg. case of updated packages but still vulnerable).
for ia, a := range addedVulnerabilities {
for ir, r := range removedVulnerabilities {
if a.ID == r.ID {
addedVulnerabilities = append(addedVulnerabilities[:ia], addedVulnerabilities[ia+1:]...)
removedVulnerabilities = append(removedVulnerabilities[:ir], removedVulnerabilities[ir+1:]...)
}
}
}
jsonhttp.Render(w, http.StatusOK, struct{ Adds, Removes []*database.Vulnerability }{Adds: addedVulnerabilities, Removes: removedVulnerabilities})
}
func getVulnerabilities(id string) []*database.Vulnerability {
layer, err := database.FindOneLayerByID(id, []string{database.FieldLayerParent}, []string{database.FieldLayerContentInstalledPackages, database.FieldLayerContentRemovedPackages})
if err != nil {
panic(err)
}
packagesNodes, err := layer.AllPackages()
if err != nil {
panic(err)
}
vulnerabilities, err := database.GetVulnerabilitiesFromLayerPackagesNodes(packagesNodes, types.Negligible, []string{database.FieldVulnerabilityID, database.FieldVulnerabilityLink, database.FieldVulnerabilityPriority, database.FieldVulnerabilityDescription})
if err != nil {
panic(err)
}
return vulnerabilities
}
// GETLayersParent returns the parent ID of a layer if it exists.
// An empty ID string is returned if the layer has no parent.
func GETLayersParent(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Find layer
layer, err := database.FindOneLayerByID(p.ByName("id"), []string{database.FieldLayerParent})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Get layer's parent.
parent, err := layer.Parent([]string{database.FieldLayerID})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
ID := ""
if parent != nil {
ID = parent.ID
}
jsonhttp.Render(w, http.StatusOK, struct{ ID string }{ID: ID})
}
// POSTVulnerabilitiesAffectedLayers returns whether the specified layers
// (by their IDs) are vulnerable to the given Vulnerability or not.
func POSTVulnerabilitiesAffectedLayers(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// Parse body.
var parameters POSTBatchLayersVulnerabilitiesParameters
if s, err := jsonhttp.ParseBody(r, ¶meters); err != nil {
jsonhttp.RenderError(w, s, err)
return
}
if len(parameters.LayersIDs) == 0 {
jsonhttp.RenderError(w, http.StatusBadRequest, errors.New("getting the entire list of affected layers is not supported yet: at least one LayerID query parameter must be provided"))
return
}
// Find vulnerability.
vulnerability, err := database.FindOneVulnerability(p.ByName("id"), []string{database.FieldVulnerabilityFixedIn})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Save the fixed in nodes into a map for fast check.
fixedInPackagesMap := make(map[string]struct{})
for _, fixedInNode := range vulnerability.FixedInNodes {
fixedInPackagesMap[fixedInNode] = struct{}{}
}
response := make(map[string]interface{})
// For each LayerID parameter.
for _, layerID := range parameters.LayersIDs {
// Find layer
layer, err := database.FindOneLayerByID(layerID, []string{database.FieldLayerParent, database.FieldLayerPackages, database.FieldLayerPackages})
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Find layer's packages.
packagesNodes, err := layer.AllPackages()
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Get successors packages of layer' packages.
successors, err := getSuccessorsFromPackagesNodes(packagesNodes)
if err != nil {
jsonhttp.RenderError(w, 0, err)
return
}
// Determine if the layer is vulnerable by verifying if one of the successors
// of its packages are fixed by the vulnerability.
vulnerable := false
for _, p := range successors {
if _, fixed := fixedInPackagesMap[p]; fixed {
vulnerable = true
break
}
}
response[layerID] = struct{ Vulnerable bool }{Vulnerable: vulnerable}
}
jsonhttp.Render(w, http.StatusOK, response)
}