Exemple #1
0
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace, err = detectNamespace(data, parent)
	if err != nil {
		return
	}
	if namespace != nil {
		log.Debugf("layer %s: Namespace is %s.", name, namespace.Name)
	} else {
		log.Debugf("layer %s: OS is unknown.", name)
	}

	// Detect features.
	features, err = detectFeatures(name, data, namespace)
	if err != nil {
		return
	}

	// If there are no feature detected, use parent's features if possible.
	// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some
	// parent's Features. It would be useful for detectors that can't find their entire result using
	// one Layer.
	if len(features) == 0 && parent != nil {
		features = parent.Features
	}

	log.Debugf("layer %s: detected %d features", name, len(features))
	return
}
Exemple #2
0
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace = detectNamespace(name, data, parent)

	// Detect features.
	featureVersions, err = detectFeatureVersions(name, data, namespace, parent)
	if err != nil {
		return
	}
	if len(featureVersions) > 0 {
		log.Debugf("layer %s: detected %d features", name, len(featureVersions))
	}

	return
}
Exemple #3
0
// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
	// Verify parameters.
	if name == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
	}

	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	if imageFormat == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a format")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
		name, utils.CleanURL(path), Version, parentName, imageFormat)

	// Check to see if the layer is already in the database.
	layer, err := datastore.FindLayer(name, false, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	if err == cerrors.ErrNotFound {
		// New layer case.
		layer = database.Layer{Name: name, EngineVersion: Version}

		// Retrieve the parent if it has one.
		// We need to get it with its Features in order to diff them.
		if parentName != "" {
			parent, err := datastore.FindLayer(parentName, true, false)
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if err == cerrors.ErrNotFound {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
					parentName)
				return ErrParentUnknown
			}
			layer.Parent = &parent
		}
	} else {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
        Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
      engine is %d. analyzing again`, name, layer.EngineVersion, Version)
	}

	// Analyze the content.
	layer.Namespace, layer.Features, err = detectContent(imageFormat, name, path, headers, layer.Parent)
	if err != nil {
		return err
	}

	return datastore.InsertLayer(layer)
}
Exemple #4
0
// Process detects the OS of a layer, the packages it installs/removes, and
// then stores everything in the database.
func Process(ID, parentID, path string) error {
	if ID == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have ID")
	}
	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s)", ID, utils.CleanURL(path), Version, parentID)

	// Check to see if the layer is already in the database.
	layer, err := database.FindOneLayerByID(ID, []string{database.FieldLayerEngineVersion})
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	var parent *database.Layer

	if layer != nil {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf("layer %s: layer content has already been processed in the past with engine %d. Current engine is %d. skipping analysis", ID, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf("layer %s: layer content has been analyzed in the past with engine %d. Current engine is %d. analyzing again", ID, layer.EngineVersion, Version)
	} else {
		// The layer is a new one, create a base struct that we will fill.
		layer = &database.Layer{ID: ID, EngineVersion: Version}

		// Check to make sure that the parent's layer has already been processed.
		if parentID != "" {
			parent, err = database.FindOneLayerByID(parentID, []string{database.FieldLayerOS, database.FieldLayerPackages, database.FieldLayerPackages})
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if parent == nil {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", ID, parentID)
				return ErrParentUnknown
			}
			layer.ParentNode = parent.GetNode()
		}
	}

	// Analyze the content.
	layer.OS, layer.InstalledPackagesNodes, layer.RemovedPackagesNodes, err = detectContent(ID, path, parent)
	if err != nil {
		return err
	}

	return database.InsertLayer(layer)
}
Exemple #5
0
// detectContent downloads a layer's archive, extracts info from it and returns
// an updated Layer struct.
//
// If parent is not nil, database.FieldLayerOS, database.FieldLayerPackages fields must be
// has been selectioned.
func detectContent(ID, path string, parent *database.Layer, imageFormat string) (OS string, installedPackagesNodes, removedPackagesNodes []string, err error) {
	data, err := getLayerData(path, imageFormat)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", ID, utils.CleanURL(path), err)
		return
	}

	OS, err = detectOS(data, parent)
	if err != nil {
		return
	}
	if OS != "" {
		log.Debugf("layer %s: OS is %s.", ID, OS)
	} else {
		log.Debugf("layer %s: OS is unknown.", ID)
	}

	packageList, err := detectors.DetectPackages(data)
	if err != nil {
		log.Errorf("layer %s: package list could not be determined: %s", ID, err)
		return
	}

	// If there are any packages, that layer modified the package list.
	if len(packageList) > 0 {
		// It is possible that the OS could not be detected, in the case of a
		// first layer setting MAINTAINER only for instance. However, if the OS
		// is unknown and packages are detected, we have to return an error.
		if OS == "" {
			log.Errorf("layer %s: OS is unknown but %d packages have been detected", ID, len(packageList))
			err = ErrUnsupported
			return
		}

		// If the layer has no parent, it can only add packages, not remove them.
		if parent == nil {
			// Build a list of the layer packages' node values.
			var installedPackages []*database.Package
			for _, p := range packageList {
				p.OS = OS
				installedPackages = append(installedPackages, p)
			}

			// Insert that list into the database.
			err = database.InsertPackages(installedPackages)
			if err != nil {
				return
			}

			// Set the InstalledPackageNodes field on content.
			for _, p := range installedPackages {
				if p.Node != "" {
					installedPackagesNodes = append(installedPackagesNodes, p.Node)
				}
			}
		} else {
			installedPackagesNodes, removedPackagesNodes, err = detectAndInsertInstalledAndRemovedPackages(OS, packageList, parent)
			if err != nil {
				return
			}
		}
	}

	log.Debugf("layer %s: detected %d packages: installs %d and removes %d packages", ID, len(packageList), len(installedPackagesNodes), len(removedPackagesNodes))
	return
}