func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { nameRef, err := reference.ParseNamed(name) if err != nil { t.Fatalf("unable to parse reference: %s", err) } ctx := context.Background() truthRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())) if err != nil { t.Fatalf("error creating registry: %v", err) } truthRepo, err := truthRegistry.Repository(ctx, nameRef) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } tr, err := truthRepo.Manifests(ctx) if err != nil { t.Fatal(err.Error()) } truthManifests := statsManifest{ manifests: tr, stats: make(map[string]int), } manifestDigest, err := populateRepo(t, ctx, truthRepo, name, tag) if err != nil { t.Fatalf(err.Error()) } localRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption) if err != nil { t.Fatalf("error creating registry: %v", err) } localRepo, err := localRegistry.Repository(ctx, nameRef) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } lr, err := localRepo.Manifests(ctx) if err != nil { t.Fatal(err.Error()) } localManifests := statsManifest{ manifests: lr, stats: make(map[string]int), } s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json") return &manifestStoreTestEnv{ manifestDigest: manifestDigest, manifests: proxyManifestStore{ ctx: ctx, localManifests: localManifests, remoteManifests: truthManifests, scheduler: s, repositoryName: nameRef, authChallenger: &mockChallenger{}, }, } }
// Populate remote store and record the digests func makeTestEnv(t *testing.T, name string) *testEnv { ctx := context.Background() truthDir, err := ioutil.TempDir("", "truth") if err != nil { t.Fatalf("unable to create tempdir: %s", err) } cacheDir, err := ioutil.TempDir("", "cache") if err != nil { t.Fatalf("unable to create tempdir: %s", err) } // todo: create a tempfile area here localRegistry, err := storage.NewRegistry(ctx, filesystem.New(truthDir), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption) if err != nil { t.Fatalf("error creating registry: %v", err) } localRepo, err := localRegistry.Repository(ctx, name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } truthRegistry, err := storage.NewRegistry(ctx, filesystem.New(cacheDir), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())) if err != nil { t.Fatalf("error creating registry: %v", err) } truthRepo, err := truthRegistry.Repository(ctx, name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } truthBlobs := statsBlobStore{ stats: make(map[string]int), blobs: truthRepo.Blobs(ctx), } localBlobs := statsBlobStore{ stats: make(map[string]int), blobs: localRepo.Blobs(ctx), } s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json") proxyBlobStore := proxyBlobStore{ remoteStore: truthBlobs, localStore: localBlobs, scheduler: s, } te := &testEnv{ store: proxyBlobStore, ctx: ctx, } return te }
func TestListener(t *testing.T) { ctx := context.Background() registry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } tl := &testListener{ ops: make(map[string]int), } repoRef, _ := reference.ParseNamed("foo/bar") repository, err := registry.Repository(ctx, repoRef) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } repository = Listen(repository, tl) // Now take the registry through a number of operations checkExerciseRepository(t, repository) expectedOps := map[string]int{ "manifest:push": 1, "manifest:pull": 1, // "manifest:delete": 0, // deletes not supported for now "layer:push": 2, "layer:pull": 2, // "layer:delete": 0, // deletes not supported for now } if !reflect.DeepEqual(tl.ops, expectedOps) { t.Fatalf("counts do not match:\n%v\n !=\n%v", tl.ops, expectedOps) } }
func createRegistry(t *testing.T, driver driver.StorageDriver) distribution.Namespace { ctx := context.Background() registry, err := storage.NewRegistry(ctx, driver, storage.EnableDelete) if err != nil { t.Fatalf("Failed to construct namespace") } return registry }
// Populate remote store and record the digests func makeTestEnv(t *testing.T, name string) testEnv { ctx := context.Background() localRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption) if err != nil { t.Fatalf("error creating registry: %v", err) } localRepo, err := localRegistry.Repository(ctx, name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } truthRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())) if err != nil { t.Fatalf("error creating registry: %v", err) } truthRepo, err := truthRegistry.Repository(ctx, name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } truthBlobs := statsBlobStore{ stats: make(map[string]int), blobs: truthRepo.Blobs(ctx), } localBlobs := statsBlobStore{ stats: make(map[string]int), blobs: localRepo.Blobs(ctx), } s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json") proxyBlobStore := proxyBlobStore{ remoteStore: truthBlobs, localStore: localBlobs, scheduler: s, } te := testEnv{ store: proxyBlobStore, ctx: ctx, } return te }
func populateTestStorage( t *testing.T, driver driver.StorageDriver, setManagedByOpenShift bool, schemaVersion int, repoImages map[string]int, testImages map[string][]*imageapi.Image, ) (map[string][]*imageapi.Image, error) { ctx := context.Background() reg, err := storage.NewRegistry(ctx, driver) if err != nil { t.Fatalf("error creating registry: %v", err) } result := make(map[string][]*imageapi.Image) for key, value := range testImages { images := make([]*imageapi.Image, len(value)) copy(images, value) result[key] = images } for imageReference := range repoImages { parsed, err := reference.Parse(imageReference) if err != nil { t.Fatalf("failed to parse reference %q: %v", imageReference, err) } namedTagged, ok := parsed.(reference.NamedTagged) if !ok { t.Fatalf("expected NamedTagged reference, not %T", parsed) } imageCount := repoImages[imageReference] for i := 0; i < imageCount; i++ { img, err := storeTestImage(ctx, reg, namedTagged, schemaVersion, setManagedByOpenShift) if err != nil { t.Fatal(err) } arr := result[imageReference] t.Logf("created image %s@%s image with layers:", namedTagged.Name(), img.Name) for _, l := range img.DockerImageLayers { t.Logf(" %s of size %d", l.Name, l.LayerSize) } result[imageReference] = append(arr, img) } } return result, nil }
func newTestRegistry( ctx context.Context, osClient client.Interface, storageDriver driver.StorageDriver, blobrepositorycachettl time.Duration, pullthrough bool, useBlobDescriptorCacheProvider bool, ) (*testRegistry, error) { if storageDriver == nil { storageDriver = inmemory.New() } dockerStorageDriver = storageDriver opts := []storage.RegistryOption{ storage.BlobDescriptorServiceFactory(&blobDescriptorServiceFactory{}), storage.EnableDelete, storage.EnableRedirect, } if useBlobDescriptorCacheProvider { cacheProvider := cache.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()) opts = append(opts, storage.BlobDescriptorCacheProvider(cacheProvider)) } reg, err := storage.NewRegistry(ctx, dockerStorageDriver, opts...) if err != nil { return nil, err } dockerRegistry = reg return &testRegistry{ Namespace: dockerRegistry, osClient: osClient, blobrepositorycachettl: blobrepositorycachettl, pullthrough: pullthrough, }, nil }
func main() { var configPath, reposPath string flag.StringVar(&configPath, "config", "", "path to a config file") flag.StringVar(&reposPath, "repos", "", "file with a list of repos") flag.Parse() if configPath == "" { fmt.Fprintln(os.Stderr, "must supply a config file with -config") flag.Usage() return } // Parse config file configFile, err := os.Open(configPath) if err != nil { panic(fmt.Sprintf("error opening config file: %v", err)) } defer configFile.Close() config, err := configuration.Parse(configFile) if err != nil { panic(fmt.Sprintf("error parsing config file: %v", err)) } ctx := context.Background() driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters()) if err != nil { panic(fmt.Sprintf("error creating storage driver: %v", err)) } registry, _ := storage.NewRegistry(ctx, driver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())) var repos []string if reposPath != "" { reposFile, err := os.Open(reposPath) if err != nil { panic(fmt.Sprintf("could not open repos file: %v", err)) } scanner := bufio.NewScanner(reposFile) for scanner.Scan() { repoName := scanner.Text() if len(repoName) > 0 { if repoName[0] == '+' { repoName = repoName[1:] } repos = append(repos, repoName) } } } else { repos = make([]string, maxRepos) n, err := registry.Repositories(ctx, repos, "") if err != nil && err != io.EOF { panic(fmt.Sprintf("unexpected error getting repo: %v", err)) } if n == maxRepos { panic("too many repositories") } repos = repos[:n] } var wg sync.WaitGroup repoChan := make(chan string) for i := 0; i < 30; i++ { wg.Add(1) go func() { for repoName := range repoChan { if err := checkRepo(registry, repoName); err != nil { fmt.Fprintln(os.Stderr, err) } } wg.Done() }() } for _, repoName := range repos { repoChan <- repoName } close(repoChan) wg.Wait() }
// TestAppDispatcher builds an application with a test dispatcher and ensures // that requests are properly dispatched and the handlers are constructed. // This only tests the dispatch mechanism. The underlying dispatchers must be // tested individually. func TestAppDispatcher(t *testing.T) { driver := inmemory.New() ctx := context.Background() registry, err := storage.NewRegistry(ctx, driver, storage.BlobDescriptorCacheProvider(memorycache.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } app := &App{ Config: &configuration.Configuration{}, Context: ctx, router: v2.Router(), driver: driver, registry: registry, } server := httptest.NewServer(app) router := v2.Router() serverURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("error parsing server url: %v", err) } varCheckingDispatcher := func(expectedVars map[string]string) dispatchFunc { return func(ctx *Context, r *http.Request) http.Handler { // Always checks the same name context if ctx.Repository.Named().Name() != getName(ctx) { t.Fatalf("unexpected name: %q != %q", ctx.Repository.Named().Name(), "foo/bar") } // Check that we have all that is expected for expectedK, expectedV := range expectedVars { if ctx.Value(expectedK) != expectedV { t.Fatalf("unexpected %s in context vars: %q != %q", expectedK, ctx.Value(expectedK), expectedV) } } // Check that we only have variables that are expected for k, v := range ctx.Value("vars").(map[string]string) { _, ok := expectedVars[k] if !ok { // name is checked on context // We have an unexpected key, fail t.Fatalf("unexpected key %q in vars with value %q", k, v) } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) } } // unflatten a list of variables, suitable for gorilla/mux, to a map[string]string unflatten := func(vars []string) map[string]string { m := make(map[string]string) for i := 0; i < len(vars)-1; i = i + 2 { m[vars[i]] = vars[i+1] } return m } for _, testcase := range []struct { endpoint string vars []string }{ { endpoint: v2.RouteNameManifest, vars: []string{ "name", "foo/bar", "reference", "sometag", }, }, { endpoint: v2.RouteNameTags, vars: []string{ "name", "foo/bar", }, }, { endpoint: v2.RouteNameBlobUpload, vars: []string{ "name", "foo/bar", }, }, { endpoint: v2.RouteNameBlobUploadChunk, vars: []string{ "name", "foo/bar", "uuid", "theuuid", }, }, } { app.register(testcase.endpoint, varCheckingDispatcher(unflatten(testcase.vars))) route := router.GetRoute(testcase.endpoint).Host(serverURL.Host) u, err := route.URL(testcase.vars...) if err != nil { t.Fatal(err) } resp, err := http.Get(u.String()) if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %v != %v", resp.StatusCode, http.StatusOK) } } }
// NewApp takes a configuration and returns a configured app, ready to serve // requests. The app only implements ServeHTTP and can be wrapped in other // handlers accordingly. func NewApp(ctx context.Context, configuration *configuration.Configuration) *App { app := &App{ Config: configuration, Context: ctx, router: v2.RouterWithPrefix(configuration.HTTP.Prefix), isCache: configuration.Proxy.RemoteURL != "", } // Register the handler dispatchers. app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) app.register(v2.RouteNameCatalog, catalogDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher) var err error app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters()) if err != nil { // TODO(stevvooe): Move the creation of a service into a protected // method, where this is created lazily. Its status can be queried via // a health check. panic(err) } purgeConfig := uploadPurgeDefaultConfig() if mc, ok := configuration.Storage["maintenance"]; ok { if v, ok := mc["uploadpurging"]; ok { purgeConfig, ok = v.(map[interface{}]interface{}) if !ok { panic("uploadpurging config key must contain additional keys") } } if v, ok := mc["readonly"]; ok { readOnly, ok := v.(map[interface{}]interface{}) if !ok { panic("readonly config key must contain additional keys") } if readOnlyEnabled, ok := readOnly["enabled"]; ok { app.readOnly, ok = readOnlyEnabled.(bool) if !ok { panic("readonly's enabled config key must have a boolean value") } } } } startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig) app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) if err != nil { panic(err) } app.configureSecret(configuration) app.configureEvents(configuration) app.configureRedis(configuration) app.configureLogHook(configuration) // Generate an ephemeral key to be used for signing converted manifests // for clients that don't support schema2. app.trustKey, err = libtrust.GenerateECP256PrivateKey() if err != nil { panic(err) } if configuration.HTTP.Host != "" { u, err := url.Parse(configuration.HTTP.Host) if err != nil { panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err)) } app.httpHost = *u } options := []storage.RegistryOption{} if app.isCache { options = append(options, storage.DisableDigestResumption) } // configure deletion if d, ok := configuration.Storage["delete"]; ok { e, ok := d["enabled"] if ok { if deleteEnabled, ok := e.(bool); ok && deleteEnabled { options = append(options, storage.EnableDelete) } } } // configure redirects var redirectDisabled bool if redirectConfig, ok := configuration.Storage["redirect"]; ok { v := redirectConfig["disable"] switch v := v.(type) { case bool: redirectDisabled = v default: panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig)) } } if redirectDisabled { ctxu.GetLogger(app).Infof("backend redirection disabled") } else { options = append(options, storage.EnableRedirect) } // configure storage caches if cc, ok := configuration.Storage["cache"]; ok { v, ok := cc["blobdescriptor"] if !ok { // Backwards compatible: "layerinfo" == "blobdescriptor" v = cc["layerinfo"] } switch v { case "redis": if app.redis == nil { panic("redis configuration required to use for layerinfo cache") } cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis) localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) if err != nil { panic("could not create registry: " + err.Error()) } ctxu.GetLogger(app).Infof("using redis blob descriptor cache") case "inmemory": cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider() localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) if err != nil { panic("could not create registry: " + err.Error()) } ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache") default: if v != "" { ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"]) } } } if app.registry == nil { // configure the registry if no cache section is available. app.registry, err = storage.NewRegistry(app.Context, app.driver, options...) if err != nil { panic("could not create registry: " + err.Error()) } } app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"]) if err != nil { panic(err) } authType := configuration.Auth.Type() if authType != "" { accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters()) if err != nil { panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err)) } app.accessController = accessController ctxu.GetLogger(app).Debugf("configured %q access controller", authType) } // configure as a pull through cache if configuration.Proxy.RemoteURL != "" { app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy) if err != nil { panic(err.Error()) } app.isCache = true ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL) } return app }
func cmdPush(c *cli.Context) { url := fmt.Sprintf("http://%s/deploy/upload", c.GlobalString("api-endpoint")) var payload struct { DeployKey string `json:"deploy_key"` } payload.DeployKey = c.String("deploy-key") var response struct { AccessKey string `json:"access_key_id"` SecretKey string `json:"secret_access_key"` Bucket string `json:"bucket"` Region string `json:"region"` Name string `json:"name"` } _, err := napping.Post(url, &payload, &response, nil) if err != nil { log.Panic("mikro-cli: push: upload-request: ", err) } options := s3.DriverParameters{ AccessKey: response.AccessKey, SecretKey: response.SecretKey, Bucket: response.Bucket, Region: aws.GetRegion(fmt.Sprint(response.Region)), ChunkSize: 5 << 21, Secure: true, } ctx := context.Background() driver, err := s3.New(options) registry, err := storage.NewRegistry(ctx, driver) imageName := c.Args()[0] repo, tag := parsers.ParseRepositoryTag(imageName) fmt.Printf("repo: %s tag: %s\n", repo, tag) bundle, err := NewImageBundle(repo, tag) if err != nil { log.Panic("mikro-cli: push: cannot export image: ", err) } defer bundle.Close() privateKey, err := libtrust.GenerateECP256PrivateKey() dgst, err := pushImageToRegistry(ctx, response.Name, c.String("commit-sha"), repo, tag, bundle, privateKey, registry) fmt.Printf("Pushed image %s as %s@%s\n", imageName, response.Name, dgst) // FIXME: we want to use the dgst here, but we can't since ECS doesn't // support it yet. registerCommit(c, c.String("deploy-key"), c.String("commit-sha"), fmt.Sprintf("private:%s:%s", response.Name, c.String("commit-sha"))) }
// Populate remote store and record the digests func makeTestEnv(t *testing.T, name string) *testEnv { nameRef, err := reference.ParseNamed(name) if err != nil { t.Fatalf("unable to parse reference: %s", err) } ctx := context.Background() truthDir, err := ioutil.TempDir("", "truth") if err != nil { t.Fatalf("unable to create tempdir: %s", err) } cacheDir, err := ioutil.TempDir("", "cache") if err != nil { t.Fatalf("unable to create tempdir: %s", err) } localDriver, err := filesystem.FromParameters(map[string]interface{}{ "rootdirectory": truthDir, }) if err != nil { t.Fatalf("unable to create filesystem driver: %s", err) } // todo: create a tempfile area here localRegistry, err := storage.NewRegistry(ctx, localDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption) if err != nil { t.Fatalf("error creating registry: %v", err) } localRepo, err := localRegistry.Repository(ctx, nameRef) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } cacheDriver, err := filesystem.FromParameters(map[string]interface{}{ "rootdirectory": cacheDir, }) if err != nil { t.Fatalf("unable to create filesystem driver: %s", err) } truthRegistry, err := storage.NewRegistry(ctx, cacheDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())) if err != nil { t.Fatalf("error creating registry: %v", err) } truthRepo, err := truthRegistry.Repository(ctx, nameRef) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } truthBlobs := statsBlobStore{ stats: make(map[string]int), blobs: truthRepo.Blobs(ctx), } localBlobs := statsBlobStore{ stats: make(map[string]int), blobs: localRepo.Blobs(ctx), } s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json") proxyBlobStore := proxyBlobStore{ repositoryName: nameRef, remoteStore: truthBlobs, localStore: localBlobs, scheduler: s, authChallenger: &mockChallenger{}, } te := &testEnv{ store: proxyBlobStore, ctx: ctx, } return te }
// NewApp takes a configuration and returns a configured app, ready to serve // requests. The app only implements ServeHTTP and can be wrapped in other // handlers accordingly. func NewApp(ctx context.Context, configuration configuration.Configuration) *App { app := &App{ Config: configuration, Context: ctx, router: v2.RouterWithPrefix(configuration.HTTP.Prefix), isCache: configuration.Proxy.RemoteURL != "", } app.Context = ctxu.WithLogger(app.Context, ctxu.GetLogger(app, "instance.id")) // Register the handler dispatchers. app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) app.register(v2.RouteNameCatalog, catalogDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher) fmt.Printf("Starting to create factory\n") var err error app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters()) if err != nil { // TODO(stevvooe): Move the creation of a service into a protected // method, where this is created lazily. Its status can be queried via // a health check. panic(err) } fmt.Printf("Created storage factory\n") purgeConfig := uploadPurgeDefaultConfig() if mc, ok := configuration.Storage["maintenance"]; ok { for k, v := range mc { switch k { case "uploadpurging": purgeConfig = v.(map[interface{}]interface{}) } } } fmt.Printf("Starting uploaduperger\n") startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig) fmt.Printf("Starting applyStorageMiddleware\n") app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) if err != nil { panic(err) } fmt.Printf("Starting configurations\n") app.configureSecret(&configuration) app.configureEvents(&configuration) app.configureRedis(&configuration) app.configureLogHook(&configuration) options := []storage.RegistryOption{} if app.isCache { options = append(options, storage.DisableDigestResumption) } // configure deletion if d, ok := configuration.Storage["delete"]; ok { e, ok := d["enabled"] if ok { if deleteEnabled, ok := e.(bool); ok && deleteEnabled { options = append(options, storage.EnableDelete) } } } // configure redirects var redirectDisabled bool if redirectConfig, ok := configuration.Storage["redirect"]; ok { v := redirectConfig["disable"] switch v := v.(type) { case bool: redirectDisabled = v default: panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig)) } } if redirectDisabled { ctxu.GetLogger(app).Infof("backend redirection disabled") } else { options = append(options, storage.EnableRedirect) } fmt.Printf("Configuring caches\n") // configure storage caches if cc, ok := configuration.Storage["cache"]; ok { v, ok := cc["blobdescriptor"] if !ok { // Backwards compatible: "layerinfo" == "blobdescriptor" v = cc["layerinfo"] } switch v { case "redis": if app.redis == nil { panic("redis configuration required to use for layerinfo cache") } cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis) localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) if err != nil { panic("could not create registry: " + err.Error()) } ctxu.GetLogger(app).Infof("using redis blob descriptor cache") case "inmemory": cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider() localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider)) app.registry, err = storage.NewRegistry(app, app.driver, localOptions...) if err != nil { panic("could not create registry: " + err.Error()) } ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache") default: if v != "" { ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"]) } } } if app.registry == nil { // configure the registry if no cache section is available. app.registry, err = storage.NewRegistry(app.Context, app.driver, options...) if err != nil { panic("could not create registry: " + err.Error()) } } fmt.Printf("Starting applyRegistryMiddleware\n") app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"]) if err != nil { panic(err) } authType := configuration.Auth.Type() if authType != "" { accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters()) if err != nil { panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err)) } app.accessController = accessController ctxu.GetLogger(app).Debugf("configured %q access controller", authType) } // configure as a pull through cache if configuration.Proxy.RemoteURL != "" { app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy) if err != nil { panic(err.Error()) } app.isCache = true ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL) } fmt.Printf("Created app\n") return app }
fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err) os.Exit(1) } ctx := context.Background() ctx, err = configureLogging(ctx, config) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err) os.Exit(1) } k, err := libtrust.GenerateECP256PrivateKey() if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } registry, err := storage.NewRegistry(ctx, driver, storage.Schema1SigningKey(k)) if err != nil { fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err) os.Exit(1) } err = storage.MarkAndSweep(ctx, driver, registry, dryRun) if err != nil { fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err) os.Exit(1) } }, }
fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err) os.Exit(1) } ctx := context.Background() ctx, err = configureLogging(ctx, config) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err) os.Exit(1) } k, err := libtrust.GenerateECP256PrivateKey() if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k)) if err != nil { fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err) os.Exit(1) } err = storage.MarkAndSweep(ctx, driver, registry, dryRun) if err != nil { fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err) os.Exit(1) } }, }
func markAndSweep(storageDriver driver.StorageDriver) error { ctx := context.Background() // Construct a registry registry, err := storage.NewRegistry(ctx, storageDriver) if err != nil { return fmt.Errorf("failed to construct registry: %v", err) } repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) if !ok { return fmt.Errorf("coercion error: unable to convert Namespace to RepositoryEnumerator") } // mark markSet := make(map[digest.Digest]struct{}) err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error { var err error named, err := reference.ParseNamed(repoName) if err != nil { return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) } repository, err := registry.Repository(ctx, named) if err != nil { return fmt.Errorf("failed to construct repository: %v", err) } manifestService, err := repository.Manifests(ctx) if err != nil { return fmt.Errorf("failed to construct manifest service: %v", err) } manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) if !ok { return fmt.Errorf("coercion error: unable to convert ManifestService into ManifestEnumerator") } err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { // Mark the manifest's blob markSet[dgst] = struct{}{} manifest, err := manifestService.Get(ctx, dgst) if err != nil { return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) } descriptors := manifest.References() for _, descriptor := range descriptors { markSet[descriptor.Digest] = struct{}{} } switch manifest.(type) { case *schema1.SignedManifest: signaturesGetter, ok := manifestService.(distribution.SignaturesGetter) if !ok { return fmt.Errorf("coercion error: unable to convert ManifestSErvice into SignaturesGetter") } signatures, err := signaturesGetter.GetSignatures(ctx, dgst) if err != nil { return fmt.Errorf("failed to get signatures for signed manifest: %v", err) } for _, signatureDigest := range signatures { markSet[signatureDigest] = struct{}{} } break case *schema2.DeserializedManifest: config := manifest.(*schema2.DeserializedManifest).Config markSet[config.Digest] = struct{}{} break } return nil }) return err }) if err != nil { return fmt.Errorf("failed to mark: %v\n", err) } // sweep blobService := registry.Blobs() deleteSet := make(map[digest.Digest]struct{}) err = blobService.Enumerate(ctx, func(dgst digest.Digest) error { // check if digest is in markSet. If not, delete it! if _, ok := markSet[dgst]; !ok { deleteSet[dgst] = struct{}{} } return nil }) // Construct vacuum vacuum := storage.NewVacuum(ctx, storageDriver) for dgst := range deleteSet { err = vacuum.RemoveBlob(string(dgst)) if err != nil { return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err) } } return err }