// CreateBaseCompilationImage will recompile the base BOSH image for a release
func (f *Fissile) CreateBaseCompilationImage(baseImageName, repository, metricsPath string, keepContainer bool) error {
if metricsPath != "" {
stampy.Stamp(metricsPath, "fissile", "create-compilation-image", "start")
defer stampy.Stamp(metricsPath, "fissile", "create-compilation-image", "done")
}
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
baseImage, err := dockerManager.FindImage(baseImageName)
if err != nil {
return fmt.Errorf("Error looking up base image %s: %s", baseImageName, err)
}
f.UI.Println(color.GreenString("Base image with ID %s found", color.YellowString(baseImage.ID)))
comp, err := compilator.NewCompilator(dockerManager, "", "", repository, compilation.UbuntuBase, f.Version, keepContainer, f.UI)
if err != nil {
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
if _, err := comp.CreateCompilationBase(baseImageName); err != nil {
return fmt.Errorf("Error creating compilation base image: %s", err.Error())
}
return nil
}
// ShowBaseImage will show details about the base BOSH images
func (f *Fissile) ShowBaseImage(repository string) error {
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
comp, err := compilator.NewCompilator(dockerManager, "", "", repository, compilation.UbuntuBase, f.Version, false, f.UI)
if err != nil {
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
image, err := dockerManager.FindImage(comp.BaseImageName())
if err != nil {
return fmt.Errorf("Error looking up base image %s: %s", comp.BaseImageName(), err.Error())
}
f.UI.Printf("\nCompilation Layer: %s\n", color.GreenString(comp.BaseImageName()))
f.UI.Printf("ID: %s\n", color.GreenString(image.ID))
f.UI.Printf("Virtual Size: %sMB\n", color.YellowString("%.2f", float64(image.VirtualSize)/(1024*1024)))
baseImageName := builder.GetBaseImageName(repository, f.Version)
image, err = dockerManager.FindImage(baseImageName)
f.UI.Printf("\nStemcell Layer: %s\n", color.GreenString(baseImageName))
f.UI.Printf("ID: %s\n", color.GreenString(image.ID))
f.UI.Printf("Virtual Size: %sMB\n", color.YellowString("%.2f", float64(image.VirtualSize)/(1024*1024)))
return nil
}
// GenerateBaseDockerImage generates a base docker image to be used as a FROM for role images
func (f *Fissile) GenerateBaseDockerImage(targetPath, baseImage, metricsPath string, noBuild bool, repository string) error {
if metricsPath != "" {
stampy.Stamp(metricsPath, "fissile", "create-role-base", "start")
defer stampy.Stamp(metricsPath, "fissile", "create-role-base", "done")
}
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
baseImageName := builder.GetBaseImageName(repository, f.Version)
image, err := dockerManager.FindImage(baseImageName)
if err == docker.ErrImageNotFound {
f.UI.Println("Image doesn't exist, it will be created ...")
} else if err != nil {
return fmt.Errorf("Error looking up image: %s", err.Error())
} else {
f.UI.Println(color.GreenString(
"Base role image %s with ID %s already exists. Doing nothing.",
color.YellowString(baseImageName),
color.YellowString(image.ID),
))
return nil
}
if !strings.HasSuffix(targetPath, string(os.PathSeparator)) {
targetPath = fmt.Sprintf("%s%c", targetPath, os.PathSeparator)
}
baseImageBuilder := builder.NewBaseImageBuilder(baseImage)
if noBuild {
f.UI.Println("Skipping image build because of flag.")
return nil
}
f.UI.Println("Building base docker image ...")
log := new(bytes.Buffer)
stdoutWriter := docker.NewFormattingWriter(
log,
docker.ColoredBuildStringFunc(baseImageName),
)
tarPopulator := baseImageBuilder.NewDockerPopulator()
err = dockerManager.BuildImageFromCallback(baseImageName, stdoutWriter, tarPopulator)
if err != nil {
log.WriteTo(f.UI)
return fmt.Errorf("Error building base image: %s", err)
}
f.UI.Println(color.GreenString("Done."))
return nil
}
// ListRoleImages lists all dev role images
func (f *Fissile) ListRoleImages(repository string, rolesManifestPath string, existingOnDocker, withVirtualSize bool) error {
if withVirtualSize && !existingOnDocker {
return fmt.Errorf("Cannot list image virtual sizes if not matching image names with docker")
}
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
var dockerManager *docker.ImageManager
var err error
if existingOnDocker {
dockerManager, err = docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
}
rolesManifest, err := model.LoadRoleManifest(rolesManifestPath, f.releases)
if err != nil {
return fmt.Errorf("Error loading roles manifest: %s", err.Error())
}
for _, role := range rolesManifest.Roles {
imageName := builder.GetRoleDevImageName(repository, role, role.GetRoleDevVersion())
if !existingOnDocker {
f.UI.Println(imageName)
continue
}
image, err := dockerManager.FindImage(imageName)
if err == docker.ErrImageNotFound {
continue
} else if err != nil {
return fmt.Errorf("Error looking up image: %s", err.Error())
}
if withVirtualSize {
f.UI.Printf(
"%s (%sMB)\n",
color.GreenString(imageName),
color.YellowString("%.2f", float64(image.VirtualSize)/(1024*1024)),
)
} else {
f.UI.Println(imageName)
}
}
return nil
}
// GeneratePackagesRoleImage builds the docker image for the packages layer
// where all packages are included
func (f *Fissile) GeneratePackagesRoleImage(repository string, roleManifest *model.RoleManifest, noBuild, force bool, packagesImageBuilder *builder.PackagesImageBuilder) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
packagesLayerImageName := packagesImageBuilder.GetRolePackageImageName(roleManifest)
if !force {
if hasImage, err := dockerManager.HasImage(packagesLayerImageName); err == nil && hasImage {
f.UI.Printf("Packages layer %s already exists. Skipping ...\n", color.YellowString(packagesLayerImageName))
return nil
}
}
baseImageName := builder.GetBaseImageName(repository, f.Version)
if hasImage, err := dockerManager.HasImage(baseImageName); err != nil {
return fmt.Errorf("Error getting base image: %s", err)
} else if !hasImage {
return fmt.Errorf("Failed to find role base %s, did you build it first?", baseImageName)
}
if noBuild {
f.UI.Println("Skipping packages layer docker image build because of --no-build flag.")
return nil
}
f.UI.Printf("Building packages layer docker image %s ...\n",
color.YellowString(packagesLayerImageName))
log := new(bytes.Buffer)
stdoutWriter := docker.NewFormattingWriter(
log,
docker.ColoredBuildStringFunc(packagesLayerImageName),
)
tarPopulator := packagesImageBuilder.NewDockerPopulator(roleManifest, force)
err = dockerManager.BuildImageFromCallback(packagesLayerImageName, stdoutWriter, tarPopulator)
if err != nil {
log.WriteTo(f.UI)
return fmt.Errorf("Error building packages layer docker image: %s", err.Error())
}
f.UI.Println(color.GreenString("Done."))
return nil
}
// determinePackagesLayerBaseImage finds the best base image to use for the
// packages layer image. Given a list of packages, it returns the base image
// name to use, as well as the set of packages that still need to be inserted.
func (p *PackagesImageBuilder) determinePackagesLayerBaseImage(packages model.Packages) (string, model.Packages, error) {
baseImageName := GetBaseImageName(p.repository, p.fissileVersion)
if baseImageOverride != "" {
baseImageName = baseImageOverride
}
var labels []string
remainingPackages := make(map[string]*model.Package, len(packages))
for _, pkg := range packages {
labels = append(labels, fmt.Sprintf("fingerprint.%s", pkg.Fingerprint))
remainingPackages[pkg.Fingerprint] = pkg
}
dockerManger, err := docker.NewImageManager()
if err != nil {
return "", nil, err
}
matchedImage, foundLabels, err := dockerManger.FindBestImageWithLabels(baseImageName, labels)
if err != nil {
return "", nil, err
}
// Find the list of packages remaining
for label := range foundLabels {
parts := strings.Split(label, ".")
if len(parts) != 2 || parts[0] != "fingerprint" {
// Should never reach here...
continue
}
delete(remainingPackages, parts[1])
}
packages = make(model.Packages, 0, len(remainingPackages))
for _, pkg := range remainingPackages {
packages = append(packages, pkg)
}
return matchedImage, packages, nil
}
// Compile will compile a list of dev BOSH releases
func (f *Fissile) Compile(repository, targetPath, roleManifestPath, metricsPath string, workerCount int) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
if metricsPath != "" {
stampy.Stamp(metricsPath, "fissile", "compile-packages", "start")
defer stampy.Stamp(metricsPath, "fissile", "compile-packages", "done")
}
dockerManager, err := docker.NewImageManager()
if err != nil {
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}
roleManifest, err := model.LoadRoleManifest(roleManifestPath, f.releases)
if err != nil {
return fmt.Errorf("Error loading roles manifest: %s", err.Error())
}
f.UI.Println(color.GreenString("Compiling packages for dev releases:"))
for _, release := range f.releases {
f.UI.Printf(" %s (%s)\n", color.YellowString(release.Name), color.MagentaString(release.Version))
}
comp, err := compilator.NewCompilator(dockerManager, targetPath, metricsPath, repository, compilation.UbuntuBase, f.Version, false, f.UI)
if err != nil {
return fmt.Errorf("Error creating a new compilator: %s", err.Error())
}
if err := comp.Compile(workerCount, f.releases, roleManifest); err != nil {
return fmt.Errorf("Error compiling packages: %s", err.Error())
}
return nil
}
func TestNewDockerPopulator(t *testing.T) {
assert := assert.New(t)
ui := termui.New(
&bytes.Buffer{},
ioutil.Discard,
nil,
)
workDir, err := os.Getwd()
assert.NoError(err)
baseImageOverride = defaultDockerTestImage
defer func() { baseImageOverride = "" }()
releasePath := filepath.Join(workDir, "../test-assets/tor-boshrelease")
releasePathCache := filepath.Join(releasePath, "bosh-cache")
compiledPackagesDir := filepath.Join(workDir, "../test-assets/tor-boshrelease-fake-compiled")
targetPath, err := ioutil.TempDir("", "fissile-test")
assert.NoError(err)
defer os.RemoveAll(targetPath)
release, err := model.NewDevRelease(releasePath, "", "", releasePathCache)
assert.NoError(err)
roleManifestPath := filepath.Join(workDir, "../test-assets/role-manifests/tor-good.yml")
rolesManifest, err := model.LoadRoleManifest(roleManifestPath, []*model.Release{release})
assert.NoError(err)
packagesImageBuilder, err := NewPackagesImageBuilder("foo", compiledPackagesDir, targetPath, "3.14.15", ui)
assert.NoError(err)
tarFile := &bytes.Buffer{}
tarPopulator := packagesImageBuilder.NewDockerPopulator(rolesManifest, false)
tarWriter := tar.NewWriter(tarFile)
assert.NoError(tarPopulator(tarWriter))
assert.NoError(tarWriter.Close())
pkg := getPackage(rolesManifest.Roles, "myrole", "tor", "tor")
if !assert.NotNil(pkg) {
return
}
// Get the docker id for the image we'll be building from...
dockerManager, err := docker.NewImageManager()
assert.NoError(err)
baseImage, err := dockerManager.FindImage(baseImageOverride)
assert.NoError(err)
// From test-assets/tor-boshrelease/dev_releases/tor/tor-0.3.5+dev.3.yml
const torFingerprint = "59523b1cc4042dff1217ab5b79ff885cdd2de032"
testFunctions := map[string]func(string){
"Dockerfile": func(contents string) {
var i int
var line string
testers := []func(){
func() { assert.Equal(fmt.Sprintf("FROM %s", baseImage.ID), line, "line 1 should start with FROM") },
func() { assert.Equal("ADD packages-src /var/vcap/packages-src/", line, "line 3 mismatch") },
func() {
expected := []string{
"LABEL",
fmt.Sprintf(`"fingerprint.%s"="libevent"`, getPackage(rolesManifest.Roles, "myrole", "tor", "libevent").Fingerprint),
fmt.Sprintf(`"fingerprint.%s"="tor"`, getPackage(rolesManifest.Roles, "myrole", "tor", "tor").Fingerprint),
}
actual := strings.Fields(line)
sort.Strings(expected[1:])
sort.Strings(actual[1:])
assert.Equal(expected, actual, "line 4 has unexpected fields")
},
}
for i, line = range getDockerfileLines(contents) {
if assert.True(i < len(testers), "Extra line #%d: %s", i+1, line) {
testers[i]()
}
}
assert.Equal(len(testers), len(getDockerfileLines(contents)), "Not enough lines")
},
"packages-src/" + torFingerprint + "/bar": func(contents string) {
assert.Empty(contents)
},
}
tarReader := tar.NewReader(tarFile)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if !assert.NoError(err) {
break
}
if tester, ok := testFunctions[header.Name]; ok {
actual, err := ioutil.ReadAll(tarReader)
assert.NoError(err)
tester(string(actual))
delete(testFunctions, header.Name)
}
}
assert.Empty(testFunctions, "Missing files in tar stream")
}
"github.com/fatih/color"
"github.com/hpcloud/termui"
workerLib "github.com/jimmysawczuk/worker"
"github.com/termie/go-shutil"
"gopkg.in/yaml.v2"
)
const (
binPrefix = "bin"
jobConfigSpecFilename = "config_spec.json"
)
var (
// newDockerImageBuilder is a stub to be replaced by the unit test
newDockerImageBuilder = func() (dockerImageBuilder, error) { return docker.NewImageManager() }
)
// dockerImageBuilder is the interface to shim around docker.RoleImageBuilder for the unit test
type dockerImageBuilder interface {
HasImage(imageName string) (bool, error)
BuildImage(dockerfileDirPath, name string, stdoutProcessor io.WriteCloser) error
}
// RoleImageBuilder represents a builder of docker role images
type RoleImageBuilder struct {
repository string
compiledPackagesPath string
targetPath string
metricsPath string
version string
