func (lb *FileBackend) BuildACI(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) { tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return "", nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(tmpDir) j, err := getJson(lb.file, layerID) if err != nil { return "", nil, fmt.Errorf("error getting image json: %v", err) } layerData := types.DockerImageData{} if err := json.Unmarshal(j, &layerData); err != nil { return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err) } tmpLayerPath := path.Join(tmpDir, layerID) tmpLayerPath += ".tar" layerFile, err := extractEmbeddedLayer(lb.file, layerID, tmpLayerPath) if err != nil { return "", nil, fmt.Errorf("error getting layer from file: %v", err) } defer layerFile.Close() log.Debug("Generating layer ACI...") aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression) if err != nil { return "", nil, fmt.Errorf("error generating ACI: %v", err) } return aciPath, manifest, nil }
func getParent(file *os.File, imgID string) (string, error) { var parent string _, err := file.Seek(0, 0) if err != nil { return "", fmt.Errorf("error seeking file: %v", err) } jsonPath := filepath.Join(imgID, "json") parentWalker := func(t *tarball.TarFile) error { if filepath.Clean(t.Name()) == jsonPath { jsonb, err := ioutil.ReadAll(t.TarStream) if err != nil { return fmt.Errorf("error reading layer json: %v", err) } var dockerData types.DockerImageData if err := json.Unmarshal(jsonb, &dockerData); err != nil { return fmt.Errorf("error unmarshaling layer data: %v", err) } parent = dockerData.Parent } return nil } tr := tar.NewReader(file) if err := tarball.Walk(*tr, parentWalker); err != nil { return "", err } log.Debug(fmt.Sprintf("Layer %q depends on layer %q", imgID, parent)) return parent, nil }
func convertPorts(dockerExposedPorts map[string]struct{}, dockerPortSpecs []string) ([]appctypes.Port, error) { ports := []appctypes.Port{} for ep := range dockerExposedPorts { appcPort, err := parseDockerPort(ep) if err != nil { return nil, err } ports = append(ports, *appcPort) } if dockerExposedPorts == nil && dockerPortSpecs != nil { log.Debug("warning: docker image uses deprecated PortSpecs field") for _, ep := range dockerPortSpecs { appcPort, err := parseDockerPort(ep) if err != nil { return nil, err } ports = append(ports, *appcPort) } } sort.Sort(appcPortSorter(ports)) return ports, nil }
func (rb *RepositoryBackend) buildACIV1(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) { tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return "", nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(tmpDir) j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData) if err != nil { return "", nil, fmt.Errorf("error getting image json: %v", err) } layerData := types.DockerImageData{} if err := json.Unmarshal(j, &layerData); err != nil { return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err) } layerFile, err := rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir) if err != nil { return "", nil, fmt.Errorf("error getting the remote layer: %v", err) } defer layerFile.Close() log.Debug("Generating layer ACI...") aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression) if err != nil { return "", nil, fmt.Errorf("error generating ACI: %v", err) } return aciPath, manifest, nil }
// squashLayers receives a list of ACI layer file names ordered from base image // to application image and squashes them into one ACI func squashLayers(images []acirenderer.Image, aciRegistry acirenderer.ACIRegistry, parsedDockerURL types.ParsedDockerURL, outputDir string, compression common.Compression) (path string, err error) { log.Debug("Squashing layers...") log.Debug("Rendering ACI...") renderedACI, err := acirenderer.GetRenderedACIFromList(images, aciRegistry) if err != nil { return "", fmt.Errorf("error rendering squashed image: %v", err) } manifests, err := getManifests(renderedACI, aciRegistry) if err != nil { return "", fmt.Errorf("error getting manifests: %v", err) } squashedFilename := getSquashedFilename(parsedDockerURL) squashedImagePath := filepath.Join(outputDir, squashedFilename) squashedTempFile, err := ioutil.TempFile(outputDir, "docker2aci-squashedFile-") if err != nil { return "", err } defer func() { if err == nil { err = squashedTempFile.Close() } else { // remove temp file on error // we ignore its error to not mask the real error os.Remove(squashedTempFile.Name()) } }() log.Debug("Writing squashed ACI...") if err := writeSquashedImage(squashedTempFile, renderedACI, aciRegistry, manifests, compression); err != nil { return "", fmt.Errorf("error writing squashed image: %v", err) } log.Debug("Validating squashed ACI...") if err := internal.ValidateACI(squashedTempFile.Name()); err != nil { return "", fmt.Errorf("error validating image: %v", err) } if err := os.Rename(squashedTempFile.Name(), squashedImagePath); err != nil { return "", err } log.Debug("ACI squashed!") return squashedImagePath, nil }
func convertReal(backend internal.Docker2ACIBackend, dockerURL string, squash bool, outputDir string, tmpDir string, compression common.Compression) ([]string, error) { log.Debug("Getting image info...") ancestry, parsedDockerURL, err := backend.GetImageInfo(dockerURL) if err != nil { return nil, err } layersOutputDir := outputDir if squash { layersOutputDir, err = ioutil.TempDir(tmpDir, "docker2aci-") if err != nil { return nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(layersOutputDir) } conversionStore := newConversionStore() var images acirenderer.Images var aciLayerPaths []string var curPwl []string for i := len(ancestry) - 1; i >= 0; i-- { layerID := ancestry[i] // only compress individual layers if we're not squashing layerCompression := compression if squash { layerCompression = common.NoCompression } aciPath, manifest, err := backend.BuildACI(i, layerID, parsedDockerURL, layersOutputDir, tmpDir, curPwl, layerCompression) if err != nil { return nil, fmt.Errorf("error building layer: %v", err) } key, err := conversionStore.WriteACI(aciPath) if err != nil { return nil, fmt.Errorf("error inserting in the conversion store: %v", err) } images = append(images, acirenderer.Image{Im: manifest, Key: key, Level: uint16(i)}) aciLayerPaths = append(aciLayerPaths, aciPath) curPwl = manifest.PathWhitelist } // acirenderer expects images in order from upper to base layer images = util.ReverseImages(images) if squash { squashedImagePath, err := squashLayers(images, conversionStore, *parsedDockerURL, outputDir, compression) if err != nil { return nil, fmt.Errorf("error squashing image: %v", err) } aciLayerPaths = []string{squashedImagePath} } return aciLayerPaths, nil }
func (lb *FileBackend) BuildACIV22(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { if len(layerIDs) < 2 { return nil, nil, fmt.Errorf("insufficient layers for oci image") } var aciLayerPaths []string var aciManifests []*schema.ImageManifest var curPwl []string imageID := layerIDs[0] layerIDs = layerIDs[1:] j, err := getJsonV22(lb.file, imageID) if err != nil { return nil, nil, fmt.Errorf("error getting layer from file: %v", err) } imageConfig := typesV2.ImageConfig{} if err := json.Unmarshal(j, &imageConfig); err != nil { return nil, nil, fmt.Errorf("error unmarshaling image data: %v", err) } tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return nil, nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(tmpDir) for i := len(layerIDs) - 1; i >= 0; i-- { parts := strings.Split(layerIDs[i], ":") tmpLayerPath := path.Join(tmpDir, parts[1]) tmpLayerPath += ".tar" layerTarPath := path.Join(append([]string{"blobs"}, parts...)...) layerFile, err := extractEmbeddedLayer(lb.file, layerTarPath, tmpLayerPath) if err != nil { return nil, nil, fmt.Errorf("error getting layer from file: %v", err) } defer layerFile.Close() log.Debug("Generating layer ACI...") var aciPath string var manifest *schema.ImageManifest if i != 0 { aciPath, manifest, err = internal.GenerateACI22LowerLayer(dockerURL, parts[1], outputDir, layerFile, curPwl, compression) } else { aciPath, manifest, err = internal.GenerateACI22TopLayer(dockerURL, &imageConfig, parts[1], outputDir, layerFile, curPwl, compression, aciManifests) } if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, manifest) curPwl = manifest.PathWhitelist } return aciLayerPaths, aciManifests, nil }
func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { if strings.Contains(layerIDs[0], ":") { return lb.BuildACIV22(layerIDs, dockerURL, outputDir, tmpBaseDir, compression) } var aciLayerPaths []string var aciManifests []*schema.ImageManifest var curPwl []string tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return nil, nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(tmpDir) for i := len(layerIDs) - 1; i >= 0; i-- { if err := common.ValidateLayerId(layerIDs[i]); err != nil { return nil, nil, err } j, err := getJson(lb.file, layerIDs[i]) if err != nil { return nil, nil, fmt.Errorf("error getting layer json: %v", err) } layerData := types.DockerImageData{} if err := json.Unmarshal(j, &layerData); err != nil { return nil, nil, fmt.Errorf("error unmarshaling layer data: %v", err) } tmpLayerPath := path.Join(tmpDir, layerIDs[i]) tmpLayerPath += ".tar" layerTarPath := path.Join(layerIDs[i], "layer.tar") layerFile, err := extractEmbeddedLayer(lb.file, layerTarPath, tmpLayerPath) if err != nil { return nil, nil, fmt.Errorf("error getting layer from file: %v", err) } defer layerFile.Close() log.Debug("Generating layer ACI...") aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression) if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, manifest) curPwl = manifest.PathWhitelist } return aciLayerPaths, aciManifests, nil }
// getAncestry computes an image ancestry, returning an ordered list // of dependencies starting from the topmost image to the base. // It checks for dependency loops via duplicate detection in the image // chain and errors out in such cases. func getAncestry(file *os.File, imgID string) ([]string, error) { var ancestry []string deps := make(map[string]bool) curImgID := imgID var err error for curImgID != "" { if deps[curImgID] { return nil, fmt.Errorf("dependency loop detected at image %q", curImgID) } deps[curImgID] = true ancestry = append(ancestry, curImgID) log.Debug(fmt.Sprintf("Getting ancestry for layer %q", curImgID)) curImgID, err = getParent(file, curImgID) if err != nil { return nil, err } } return ancestry, nil }
func (rb *RepositoryBackend) buildACIV2(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) { manifest := rb.imageManifests[*dockerURL] layerIndex, err := getLayerIndex(layerID, manifest) if err != nil { return "", nil, err } if len(manifest.History) <= layerIndex { return "", nil, fmt.Errorf("history not found for layer %s", layerID) } layerData := types.DockerImageData{} if err := json.Unmarshal([]byte(manifest.History[layerIndex].V1Compatibility), &layerData); err != nil { return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err) } tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return "", nil, fmt.Errorf("error creating dir: %v", err) } defer os.RemoveAll(tmpDir) layerFile, err := rb.getLayerV2(layerID, dockerURL, tmpDir) if err != nil { return "", nil, fmt.Errorf("error getting the remote layer: %v", err) } defer layerFile.Close() log.Debug("Generating layer ACI...") aciPath, aciManifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression) if err != nil { return "", nil, fmt.Errorf("error generating ACI: %v", err) } return aciPath, aciManifest, nil }
func getImageID(file *os.File, dockerURL *types.ParsedDockerURL, name string) (string, []string, *types.ParsedDockerURL, error) { log.Debug("getting image id...") type tags map[string]string type apps map[string]tags _, err := file.Seek(0, 0) if err != nil { return "", nil, nil, fmt.Errorf("error seeking file: %v", err) } tag := "latest" if dockerURL != nil { tag = dockerURL.Tag } var imageID string var ancestry []string var appName string reposWalker := func(t *tarball.TarFile) error { clean := filepath.Clean(t.Name()) if clean == "repositories" { repob, err := ioutil.ReadAll(t.TarStream) if err != nil { return fmt.Errorf("error reading repositories file: %v", err) } var repositories apps if err := json.Unmarshal(repob, &repositories); err != nil { return fmt.Errorf("error unmarshaling repositories file") } if dockerURL == nil { n := len(repositories) switch { case n == 1: for key, _ := range repositories { appName = key } case n > 1: var appNames []string for key, _ := range repositories { appNames = append(appNames, key) } return &common.ErrSeveralImages{ Msg: "several images found", Images: appNames, } default: return fmt.Errorf("no images found") } } else { appName = dockerURL.ImageName } app, ok := repositories[appName] if !ok { return fmt.Errorf("app %q not found", appName) } _, ok = app[tag] if !ok { if len(app) == 1 { for key, _ := range app { tag = key } } else { return fmt.Errorf("tag %q not found", tag) } } if dockerURL == nil { dockerURL = &types.ParsedDockerURL{ IndexURL: "", Tag: tag, ImageName: appName, } } imageID = string(app[tag]) } if clean == "refs/"+tag { refb, err := ioutil.ReadAll(t.TarStream) if err != nil { return fmt.Errorf("error reading ref descriptor for tag %s: %v", tag, err) } if dockerURL == nil { dockerURL = &types.ParsedDockerURL{ IndexURL: "", Tag: tag, ImageName: name, } } var ref spec.Descriptor if err := json.Unmarshal(refb, &ref); err != nil { return fmt.Errorf("error unmarshaling ref descriptor for tag %s", tag) } imageID, ancestry, err = getDataFromManifest(file, ref.Digest) if err != nil { return err } return io.EOF } return nil } tr := tar.NewReader(file) if err := tarball.Walk(*tr, reposWalker); err != nil && err != io.EOF { return "", nil, nil, err } if imageID == "" { return "", nil, nil, fmt.Errorf("Could not find image") } return imageID, ancestry, dockerURL, nil }
func (rb *RepositoryBackend) buildACIV21(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { layerFiles := make([]*os.File, len(layerIDs)) layerDatas := make([]types.DockerImageData, len(layerIDs)) tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return nil, nil, err } defer os.RemoveAll(tmpParentDir) copier := progressutil.NewCopyProgressPrinter() var errChannels []chan error closers := make([]io.ReadCloser, len(layerIDs)) var wg sync.WaitGroup for i, layerID := range layerIDs { wg.Add(1) errChan := make(chan error, 1) errChannels = append(errChannels, errChan) // https://github.com/golang/go/wiki/CommonMistakes i := i // golang-- layerID := layerID go func() { defer wg.Done() manifest := rb.imageManifests[*dockerURL] layerIndex, err := getLayerIndex(layerID, manifest) if err != nil { errChan <- err return } if len(manifest.History) <= layerIndex { errChan <- fmt.Errorf("history not found for layer %s", layerID) return } layerDatas[i] = types.DockerImageData{} if err := json.Unmarshal([]byte(manifest.History[layerIndex].V1Compatibility), &layerDatas[i]); err != nil { errChan <- fmt.Errorf("error unmarshaling layer data: %v", err) return } tmpDir, err := ioutil.TempDir(tmpParentDir, "") if err != nil { errChan <- fmt.Errorf("error creating dir: %v", err) return } layerFiles[i], closers[i], err = rb.getLayerV2(layerID, dockerURL, tmpDir, copier) if err != nil { errChan <- fmt.Errorf("error getting the remote layer: %v", err) return } errChan <- nil }() } // Need to wait for all of the readers to be added to the copier (which happens during rb.getLayerV2) wg.Wait() err = copier.PrintAndWait(os.Stderr, 500*time.Millisecond, nil) if err != nil { return nil, nil, err } for _, closer := range closers { if closer != nil { closer.Close() } } for _, errChan := range errChannels { err := <-errChan if err != nil { return nil, nil, err } } for _, layerFile := range layerFiles { err := layerFile.Sync() if err != nil { return nil, nil, err } } var aciLayerPaths []string var aciManifests []*schema.ImageManifest var curPwl []string for i := len(layerIDs) - 1; i >= 0; i-- { log.Debug("Generating layer ACI...") aciPath, aciManifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression) if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, aciManifest) curPwl = aciManifest.PathWhitelist layerFiles[i].Close() } return aciLayerPaths, aciManifests, nil }
func (rb *RepositoryBackend) buildACIV22(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { layerFiles := make([]*os.File, len(layerIDs)) tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return nil, nil, err } defer os.RemoveAll(tmpParentDir) copier := progressutil.NewCopyProgressPrinter() resultChan := make(chan layer, len(layerIDs)) for i, layerID := range layerIDs { // https://github.com/golang/go/wiki/CommonMistakes i := i // golang-- layerID := layerID go func() { tmpDir, err := ioutil.TempDir(tmpParentDir, "") if err != nil { resultChan <- layer{ index: i, err: fmt.Errorf("error creating dir: %v", err), } return } layerFile, closer, err := rb.getLayerV2(layerID, dockerURL, tmpDir, copier) if err != nil { resultChan <- layer{ index: i, err: fmt.Errorf("error getting the remote layer: %v", err), } return } resultChan <- layer{ index: i, file: layerFile, closer: closer, err: nil, } }() } var errs []error for i := 0; i < len(layerIDs); i++ { res := <-resultChan if res.closer != nil { defer res.closer.Close() } if res.file != nil { defer res.file.Close() } if res.err != nil { errs = append(errs, res.err) } layerFiles[res.index] = res.file } if len(errs) > 0 { return nil, nil, errs[0] } err = copier.PrintAndWait(os.Stderr, 500*time.Millisecond, nil) if err != nil { return nil, nil, err } for _, layerFile := range layerFiles { err := layerFile.Sync() if err != nil { return nil, nil, err } } var aciLayerPaths []string var aciManifests []*schema.ImageManifest var curPwl []string for i := len(layerIDs) - 1; i > 0; i-- { log.Debug("Generating layer ACI...") aciPath, aciManifest, err := internal.GenerateACI22LowerLayer(dockerURL, layerIDs[i], outputDir, layerFiles[i], curPwl, compression) if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, aciManifest) curPwl = aciManifest.PathWhitelist } aciPath, aciManifest, err := internal.GenerateACI22TopLayer(dockerURL, rb.imageConfigs[*dockerURL], layerIDs[0], outputDir, layerFiles[0], curPwl, compression, aciManifests) if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, aciManifest) return aciLayerPaths, aciManifests, nil }
func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { layerFiles := make([]*os.File, len(layerIDs)) layerDatas := make([]types.DockerImageData, len(layerIDs)) tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") if err != nil { return nil, nil, err } defer os.RemoveAll(tmpParentDir) var doneChannels []chan error for i, layerID := range layerIDs { doneChan := make(chan error) doneChannels = append(doneChannels, doneChan) // https://github.com/golang/go/wiki/CommonMistakes i := i // golang-- layerID := layerID go func() { tmpDir, err := ioutil.TempDir(tmpParentDir, "") if err != nil { doneChan <- fmt.Errorf("error creating dir: %v", err) return } j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData) if err != nil { doneChan <- fmt.Errorf("error getting image json: %v", err) return } layerDatas[i] = types.DockerImageData{} if err := json.Unmarshal(j, &layerDatas[i]); err != nil { doneChan <- fmt.Errorf("error unmarshaling layer data: %v", err) return } layerFiles[i], err = rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir) if err != nil { doneChan <- fmt.Errorf("error getting the remote layer: %v", err) return } doneChan <- nil }() } for _, doneChan := range doneChannels { err := <-doneChan if err != nil { return nil, nil, err } } var aciLayerPaths []string var aciManifests []*schema.ImageManifest var curPwl []string for i := len(layerIDs) - 1; i >= 0; i-- { log.Debug("Generating layer ACI...") aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression) if err != nil { return nil, nil, fmt.Errorf("error generating ACI: %v", err) } aciLayerPaths = append(aciLayerPaths, aciPath) aciManifests = append(aciManifests, manifest) curPwl = manifest.PathWhitelist layerFiles[i].Close() } return aciLayerPaths, aciManifests, nil }