func (mf *v2ManifestFetcher) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (img *image.Image, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return nil, "", err
}
target := mfst.Target()
configChan := make(chan []byte, 1)
errChan := make(chan error, 1)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
// Pull the image config
go func() {
configJSON, err := mf.pullSchema2ImageConfig(ctx, target.Digest)
if err != nil {
errChan <- err
cancel()
return
}
configChan <- configJSON
}()
var (
configJSON []byte // raw serialized image config
unmarshalledConfig image.Image // deserialized image config
)
if runtime.GOOS == "windows" {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return nil, "", err
}
if unmarshalledConfig.RootFS == nil {
return nil, "", errors.New("image config has no rootfs section")
}
}
if configJSON == nil {
configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan)
if err != nil {
return nil, "", err
}
}
img, err = image.NewFromJSON(configJSON)
if err != nil {
return nil, "", err
}
return img, manifestDigest, nil
}
func (mf *v2ManifestFetcher) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (img *image.Image, manifestDigest digest.Digest, err error) {
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
if err != nil {
return nil, "", err
}
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil {
return nil, "", err
}
// Image history converted to the new format
var history []image.History
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil {
return nil, "", err
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil {
return nil, "", err
}
history = append(history, h)
}
rootFS := image.NewRootFS()
configRaw, err := makeRawConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history)
config, err := json.Marshal(configRaw)
if err != nil {
return nil, "", err
}
img, err = image.NewFromJSON(config)
if err != nil {
return nil, "", err
}
manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
return img, manifestDigest, nil
}
func TestMakeV1ConfigFromConfig(t *testing.T) {
img := &image.Image{
V1Image: image.V1Image{
ID: "v2id",
Parent: "v2parent",
OS: "os",
},
OSVersion: "osversion",
RootFS: &image.RootFS{
Type: "layers",
},
}
v2js, err := json.Marshal(img)
if err != nil {
t.Fatal(err)
}
// Convert the image back in order to get RawJSON() support.
img, err = image.NewFromJSON(v2js)
if err != nil {
t.Fatal(err)
}
js, err := MakeV1ConfigFromConfig(img, "v1id", "v1parent", false)
if err != nil {
t.Fatal(err)
}
newimg := &image.Image{}
err = json.Unmarshal(js, newimg)
if err != nil {
t.Fatal(err)
}
if newimg.V1Image.ID != "v1id" || newimg.Parent != "v1parent" {
t.Error("ids should have changed", newimg.V1Image.ID, newimg.V1Image.Parent)
}
if newimg.RootFS != nil {
t.Error("rootfs should have been removed")
}
if newimg.V1Image.OS != "os" {
t.Error("os should have been preserved")
}
}
func (mf *v1ManifestFetcher) pullImageJSON(imgID, endpoint string, token []string) (*image.Image, error) {
imgJSON, _, err := mf.session.GetRemoteImageJSON(imgID, endpoint)
if err != nil {
return nil, err
}
h, err := v1.HistoryFromConfig(imgJSON, false)
if err != nil {
return nil, err
}
configRaw, err := makeRawConfigFromV1Config(imgJSON, image.NewRootFS(), []image.History{h})
if err != nil {
return nil, err
}
config, err := json.Marshal(configRaw)
if err != nil {
return nil, err
}
img, err := image.NewFromJSON(config)
if err != nil {
return nil, err
}
return img, nil
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
var (
sf = streamformatter.NewJSONStreamFormatter()
progressOutput progress.Output
)
if !quiet {
progressOutput = sf.NewProgressOutput(outStream, false)
outStream = &streamformatter.StdoutFormatter{Writer: outStream, StreamFormatter: streamformatter.NewJSONStreamFormatter()}
}
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream, progressOutput)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
var parentLinks []parentLink
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
r := rootFS
r.Append(diffID)
newLayer, err := l.ls.Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), progressOutput)
if err != nil {
return err
}
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
parentLinks = append(parentLinks, parentLink{imgID, m.Parent})
l.loggerImgEvent.LogImageEvent(imgID.String(), imgID.String(), "load")
}
for _, p := range validatedParentLinks(parentLinks) {
if p.parentID != "" {
if err := l.setParentID(p.id, p.parentID); err != nil {
return err
}
}
}
return nil
}
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
tmpDir, err := ioutil.TempDir("", "docker-import-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
return err
}
// read manifest, if no file then load in legacy mode
manifestPath, err := safePath(tmpDir, manifestFileName)
if err != nil {
return err
}
manifestFile, err := os.Open(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return l.legacyLoad(tmpDir, outStream)
}
return manifestFile.Close()
}
defer manifestFile.Close()
var manifest []manifestItem
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
return err
}
for _, m := range manifest {
configPath, err := safePath(tmpDir, m.Config)
if err != nil {
return err
}
config, err := ioutil.ReadFile(configPath)
if err != nil {
return err
}
img, err := image.NewFromJSON(config)
if err != nil {
return err
}
var rootFS image.RootFS
rootFS = *img.RootFS
rootFS.DiffIDs = nil
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
}
for i, diffID := range img.RootFS.DiffIDs {
layerPath, err := safePath(tmpDir, m.Layers[i])
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, rootFS)
if err != nil {
return err
}
defer layer.ReleaseAndLog(l.ls, newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
rootFS.Append(diffID)
}
imgID, err := l.is.Create(config)
if err != nil {
return err
}
for _, repoTag := range m.RepoTags {
named, err := reference.ParseNamed(repoTag)
if err != nil {
return err
}
ref, ok := named.(reference.NamedTagged)
if !ok {
return fmt.Errorf("invalid tag %q", repoTag)
}
l.setLoadedTag(ref, imgID, outStream)
}
}
return nil
}
func TestCreateV2Manifest(t *testing.T) {
imgJSON := `{
"architecture": "amd64",
"config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"echo hi"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
"container_config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"created": "2015-11-04T23:06:32.365666163Z",
"docker_version": "1.9.0-dev",
"history": [
{
"created": "2015-10-31T22:22:54.690851953Z",
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
},
{
"created": "2015-10-31T22:22:55.613815829Z",
"created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
},
{
"created": "2015-11-04T23:06:30.934316144Z",
"created_by": "/bin/sh -c #(nop) ENV derived=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:31.192097572Z",
"created_by": "/bin/sh -c #(nop) ENV asdf=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:32.083868454Z",
"created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
},
{
"created": "2015-11-04T23:06:32.365666163Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"diff_ids": [
"sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
"sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
],
"type": "layers"
}
}`
// To fill in rawJSON
img, err := image.NewFromJSON([]byte(imgJSON))
if err != nil {
t.Fatalf("json decoding failed: %v", err)
}
fsLayers := map[layer.DiffID]digest.Digest{
layer.DiffID("sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1"): digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
layer.DiffID("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"): digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
layer.DiffID("sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"): digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
}
manifest, err := CreateV2Manifest("testrepo", "testtag", img, fsLayers)
if err != nil {
t.Fatalf("CreateV2Manifest returned error: %v", err)
}
if manifest.Versioned.SchemaVersion != 1 {
t.Fatal("SchemaVersion != 1")
}
if manifest.Name != "testrepo" {
t.Fatal("incorrect name in manifest")
}
if manifest.Tag != "testtag" {
t.Fatal("incorrect tag in manifest")
}
if manifest.Architecture != "amd64" {
t.Fatal("incorrect arch in manifest")
}
expectedFSLayers := []schema1.FSLayer{
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
}
if len(manifest.FSLayers) != len(expectedFSLayers) {
t.Fatalf("wrong number of FSLayers: %d", len(manifest.FSLayers))
}
if !reflect.DeepEqual(manifest.FSLayers, expectedFSLayers) {
t.Fatal("wrong FSLayers list")
}
expectedV1Compatibility := []string{
`{"architecture":"amd64","config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","echo hi"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"container":"e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001","container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"created":"2015-11-04T23:06:32.365666163Z","docker_version":"1.9.0-dev","id":"d728140d3fd23dfcac505954af0b2224b3579b177029eded62916579eb19ac64","os":"linux","parent":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","throwaway":true}`,
`{"id":"0594e66a9830fa5ba73b66349eb221ea4beb6bac8d2148b90a0f371f8d67bcd5","parent":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","created":"2015-11-04T23:06:32.083868454Z","container_config":{"Cmd":["/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"]}}`,
`{"id":"39bc0dbed47060dd8952b048e73744ae471fe50354d2c267d308292c53b83ce1","parent":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","created":"2015-11-04T23:06:31.192097572Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV asdf=true"]},"throwaway":true}`,
`{"id":"875d7f206c023dc979e1677567a01364074f82b61e220c9b83a4610170490381","parent":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","created":"2015-11-04T23:06:30.934316144Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV derived=true"]},"throwaway":true}`,
`{"id":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","parent":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:55.613815829Z","container_config":{"Cmd":["/bin/sh -c #(nop) CMD [\"sh\"]"]}}`,
`{"id":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:54.690851953Z","container_config":{"Cmd":["/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"]}}`,
}
if len(manifest.History) != len(expectedV1Compatibility) {
t.Fatalf("wrong number of history entries: %d", len(manifest.History))
}
for i := range expectedV1Compatibility {
if manifest.History[i].V1Compatibility != expectedV1Compatibility[i] {
t.Fatalf("wrong V1Compatibility %d. expected:\n%s\ngot:\n%s", i, expectedV1Compatibility[i], manifest.History[i].V1Compatibility)
}
}
}