// newPuller returns a Puller interface that will pull from either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 puller will be created. The other parameters are passed
// through to the underlying puller implementation for use during the actual
// pull operation.
func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePullConfig *ImagePullConfig) (Puller, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Puller{
blobSumService: metadata.NewBlobSumService(imagePullConfig.MetadataStore),
endpoint: endpoint,
config: imagePullConfig,
repoInfo: repoInfo,
}, nil
case registry.APIVersion1:
return &v1Puller{
v1IDService: metadata.NewV1IDService(imagePullConfig.MetadataStore),
endpoint: endpoint,
config: imagePullConfig,
repoInfo: repoInfo,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
// NewPusher creates a new Pusher interface that will push to either a v1 or v2
// registry. The endpoint argument contains a Version field that determines
// whether a v1 or v2 pusher will be created. The other parameters are passed
// through to the underlying pusher implementation for use during the actual
// push operation.
func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, imagePushConfig *ImagePushConfig) (Pusher, error) {
switch endpoint.Version {
case registry.APIVersion2:
return &v2Pusher{
blobSumService: metadata.NewBlobSumService(imagePushConfig.MetadataStore),
ref: ref,
endpoint: endpoint,
repoInfo: repoInfo,
config: imagePushConfig,
layersPushed: pushMap{layersPushed: make(map[digest.Digest]bool)},
}, nil
case registry.APIVersion1:
return &v1Pusher{
v1IDService: metadata.NewV1IDService(imagePushConfig.MetadataStore),
ref: ref,
endpoint: endpoint,
repoInfo: repoInfo,
config: imagePushConfig,
}, nil
}
return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
}
func TestMigrateImages(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "migrate-images")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
// busybox from 1.9
id1, err := addImage(tmpdir, `{"architecture":"amd64","config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"23304fc829f9b9349416f6eb1afec162907eba3a328f51d53a17f8986f865d65","container_config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"],"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-10-31T22:22:54.690851953Z","docker_version":"1.8.2","layer_id":"sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57","os":"linux"}`, "", "")
if err != nil {
t.Fatal(err)
}
id2, err := addImage(tmpdir, `{"architecture":"amd64","config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["sh"],"Image":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"349b014153779e30093d94f6df2a43c7a0a164e05aa207389917b540add39b51","container_config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) CMD [\"sh\"]"],"Image":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-10-31T22:22:55.613815829Z","docker_version":"1.8.2","layer_id":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","os":"linux","parent_id":"sha256:039b63dd2cbaa10d6015ea574392530571ed8d7b174090f032211285a71881d0"}`, id1, "")
if err != nil {
t.Fatal(err)
}
ls := &mockRegistrar{}
ifs, err := image.NewFSStoreBackend(filepath.Join(tmpdir, "imagedb"))
if err != nil {
t.Fatal(err)
}
is, err := image.NewImageStore(ifs, ls)
if err != nil {
t.Fatal(err)
}
ms, err := metadata.NewFSMetadataStore(filepath.Join(tmpdir, "distribution"))
if err != nil {
t.Fatal(err)
}
mappings := make(map[string]image.ID)
err = migrateImages(tmpdir, ls, is, ms, mappings)
if err != nil {
t.Fatal(err)
}
expected := map[string]image.ID{
id1: image.ID("sha256:ca406eaf9c26898414ff5b7b3a023c33310759d6203be0663dbf1b3a712f432d"),
id2: image.ID("sha256:a488bec94bb96b26a968f913d25ef7d8d204d727ca328b52b4b059c7d03260b6"),
}
if !reflect.DeepEqual(mappings, expected) {
t.Fatalf("invalid image mappings: expected %q, got %q", expected, mappings)
}
if actual, expected := ls.count, 2; actual != expected {
t.Fatalf("invalid register count: expected %q, got %q", expected, actual)
}
ls.count = 0
// next images are busybox from 1.8.2
_, err = addImage(tmpdir, `{"id":"17583c7dd0dae6244203b8029733bdb7d17fccbb2b5d93e2b24cf48b8bfd06e2","parent":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","created":"2015-10-31T22:22:55.613815829Z","container":"349b014153779e30093d94f6df2a43c7a0a164e05aa207389917b540add39b51","container_config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"PublishService":"","Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) CMD [\"sh\"]"],"Image":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"MacAddress":"","OnBuild":null,"Labels":null},"docker_version":"1.8.2","config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"PublishService":"","Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["sh"],"Image":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"MacAddress":"","OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux","Size":0}`, "", "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
if err != nil {
t.Fatal(err)
}
_, err = addImage(tmpdir, `{"id":"d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498","created":"2015-10-31T22:22:54.690851953Z","container":"23304fc829f9b9349416f6eb1afec162907eba3a328f51d53a17f8986f865d65","container_config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"PublishService":"","Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"],"Image":"","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"MacAddress":"","OnBuild":null,"Labels":null},"docker_version":"1.8.2","config":{"Hostname":"23304fc829f9","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"ExposedPorts":null,"PublishService":"","Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"MacAddress":"","OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux","Size":1108935}`, "", "sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57")
if err != nil {
t.Fatal(err)
}
err = migrateImages(tmpdir, ls, is, ms, mappings)
if err != nil {
t.Fatal(err)
}
expected["d1592a710ac323612bd786fa8ac20727c58d8a67847e5a65177c594f43919498"] = image.ID("sha256:c091bb33854e57e6902b74c08719856d30b5593c7db6143b2b48376b8a588395")
expected["17583c7dd0dae6244203b8029733bdb7d17fccbb2b5d93e2b24cf48b8bfd06e2"] = image.ID("sha256:d963020e755ff2715b936065949472c1f8a6300144b922992a1a421999e71f07")
if actual, expected := ls.count, 2; actual != expected {
t.Fatalf("invalid register count: expected %q, got %q", expected, actual)
}
blobSumService := metadata.NewBlobSumService(ms)
blobsums, err := blobSumService.GetBlobSums(layer.EmptyLayer.DiffID())
if err != nil {
t.Fatal(err)
}
expectedBlobsums := []digest.Digest{
"sha256:55dc925c23d1ed82551fd018c27ac3ee731377b6bad3963a2a4e76e753d70e57",
"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4",
}
if !reflect.DeepEqual(expectedBlobsums, blobsums) {
t.Fatalf("invalid blobsums: expected %q, got %q", expectedBlobsums, blobsums)
}
}
func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
defer func() {
if err != nil {
logrus.Errorf("migration failed for %v, err: %v", id, err)
}
}()
jsonFile := filepath.Join(root, graphDirName, id, "json")
imageJSON, err := ioutil.ReadFile(jsonFile)
if err != nil {
return err
}
var parent struct {
Parent string
ParentID digest.Digest `json:"parent_id"`
}
if err := json.Unmarshal(imageJSON, &parent); err != nil {
return err
}
if parent.Parent == "" && parent.ParentID != "" { // v1.9
parent.Parent = parent.ParentID.Hex()
}
// compatibilityID for parent
parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "parent"))
if err == nil && len(parentCompatibilityID) > 0 {
parent.Parent = string(parentCompatibilityID)
}
var parentID image.ID
if parent.Parent != "" {
var exists bool
if parentID, exists = mappings[parent.Parent]; !exists {
if err := migrateImage(parent.Parent, root, ls, is, ms, mappings); err != nil {
// todo: fail or allow broken chains?
return err
}
parentID = mappings[parent.Parent]
}
}
rootFS := image.NewRootFS()
var history []image.History
if parentID != "" {
parentImg, err := is.Get(parentID)
if err != nil {
return err
}
rootFS = parentImg.RootFS
history = parentImg.History
}
layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), filepath.Join(filepath.Join(root, graphDirName, id, tarDataFileName)))
if err != nil {
return err
}
logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
h, err := imagev1.HistoryFromConfig(imageJSON, false)
if err != nil {
return err
}
history = append(history, h)
rootFS.Append(layer.DiffID())
config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
if err != nil {
return err
}
strongID, err := is.Create(config)
if err != nil {
return err
}
logrus.Infof("migrated image %s to %s", id, strongID)
if parentID != "" {
if err := is.SetParent(strongID, parentID); err != nil {
return err
}
}
checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
if err == nil { // best effort
dgst, err := digest.ParseDigest(string(checksum))
if err == nil {
blobSumService := metadata.NewBlobSumService(ms)
blobSumService.Add(layer.DiffID(), dgst)
}
}
_, err = ls.Release(layer)
if err != nil {
return err
}
mappings[id] = strongID
return
}
func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) {
defer func() {
if err != nil {
logrus.Errorf("migration failed for %v, err: %v", id, err)
}
}()
parent, err := getParent(filepath.Join(root, graphDirName, id))
if err != nil {
return err
}
var parentID image.ID
if parent != "" {
var exists bool
if parentID, exists = mappings[parent]; !exists {
if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil {
// todo: fail or allow broken chains?
return err
}
parentID = mappings[parent]
}
}
rootFS := image.NewRootFS()
var history []image.History
if parentID != "" {
parentImg, err := is.Get(parentID)
if err != nil {
return err
}
rootFS = parentImg.RootFS
history = parentImg.History
}
diffID, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName))
if err != nil {
return err
}
sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName))
if err != nil {
return err
}
size, err := strconv.ParseInt(string(sizeStr), 10, 64)
if err != nil {
return err
}
layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size)
if err != nil {
return err
}
logrus.Infof("migrated layer %s to %s", id, layer.DiffID())
jsonFile := filepath.Join(root, graphDirName, id, "json")
imageJSON, err := ioutil.ReadFile(jsonFile)
if err != nil {
return err
}
h, err := imagev1.HistoryFromConfig(imageJSON, false)
if err != nil {
return err
}
history = append(history, h)
rootFS.Append(layer.DiffID())
config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history)
if err != nil {
return err
}
strongID, err := is.Create(config)
if err != nil {
return err
}
logrus.Infof("migrated image %s to %s", id, strongID)
if parentID != "" {
if err := is.SetParent(strongID, parentID); err != nil {
return err
}
}
checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum"))
if err == nil { // best effort
dgst, err := digest.ParseDigest(string(checksum))
if err == nil {
blobSumService := metadata.NewBlobSumService(ms)
blobSumService.Add(layer.DiffID(), dgst)
}
}
_, err = ls.Release(layer)
if err != nil {
return err
}
mappings[id] = strongID
return
}