func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
nameRef, err := reference.ParseNamed(name)
if err != nil {
t.Fatalf("unable to parse reference: %s", err)
}
ctx := context.Background()
truthRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
tr, err := truthRepo.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
truthManifests := statsManifest{
manifests: tr,
stats: make(map[string]int),
}
manifestDigest, err := populateRepo(t, ctx, truthRepo, name, tag)
if err != nil {
t.Fatalf(err.Error())
}
localRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
lr, err := localRepo.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
localManifests := statsManifest{
manifests: lr,
stats: make(map[string]int),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
return &manifestStoreTestEnv{
manifestDigest: manifestDigest,
manifests: proxyManifestStore{
ctx: ctx,
localManifests: localManifests,
remoteManifests: truthManifests,
scheduler: s,
repositoryName: nameRef,
authChallenger: &mockChallenger{},
},
}
}
// Populate remote store and record the digests
func makeTestEnv(t *testing.T, name string) *testEnv {
ctx := context.Background()
truthDir, err := ioutil.TempDir("", "truth")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
cacheDir, err := ioutil.TempDir("", "cache")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
// todo: create a tempfile area here
localRegistry, err := storage.NewRegistry(ctx, filesystem.New(truthDir), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthRegistry, err := storage.NewRegistry(ctx, filesystem.New(cacheDir), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: truthRepo.Blobs(ctx),
}
localBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: localRepo.Blobs(ctx),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
proxyBlobStore := proxyBlobStore{
remoteStore: truthBlobs,
localStore: localBlobs,
scheduler: s,
}
te := &testEnv{
store: proxyBlobStore,
ctx: ctx,
}
return te
}
func TestListener(t *testing.T) {
ctx := context.Background()
registry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
tl := &testListener{
ops: make(map[string]int),
}
repoRef, _ := reference.ParseNamed("foo/bar")
repository, err := registry.Repository(ctx, repoRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
repository = Listen(repository, tl)
// Now take the registry through a number of operations
checkExerciseRepository(t, repository)
expectedOps := map[string]int{
"manifest:push": 1,
"manifest:pull": 1,
// "manifest:delete": 0, // deletes not supported for now
"layer:push": 2,
"layer:pull": 2,
// "layer:delete": 0, // deletes not supported for now
}
if !reflect.DeepEqual(tl.ops, expectedOps) {
t.Fatalf("counts do not match:\n%v\n !=\n%v", tl.ops, expectedOps)
}
}
func createRegistry(t *testing.T, driver driver.StorageDriver) distribution.Namespace {
ctx := context.Background()
registry, err := storage.NewRegistry(ctx, driver, storage.EnableDelete)
if err != nil {
t.Fatalf("Failed to construct namespace")
}
return registry
}
// Populate remote store and record the digests
func makeTestEnv(t *testing.T, name string) testEnv {
ctx := context.Background()
localRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthRegistry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: truthRepo.Blobs(ctx),
}
localBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: localRepo.Blobs(ctx),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
proxyBlobStore := proxyBlobStore{
remoteStore: truthBlobs,
localStore: localBlobs,
scheduler: s,
}
te := testEnv{
store: proxyBlobStore,
ctx: ctx,
}
return te
}
func populateTestStorage(
t *testing.T,
driver driver.StorageDriver,
setManagedByOpenShift bool,
schemaVersion int,
repoImages map[string]int,
testImages map[string][]*imageapi.Image,
) (map[string][]*imageapi.Image, error) {
ctx := context.Background()
reg, err := storage.NewRegistry(ctx, driver)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
result := make(map[string][]*imageapi.Image)
for key, value := range testImages {
images := make([]*imageapi.Image, len(value))
copy(images, value)
result[key] = images
}
for imageReference := range repoImages {
parsed, err := reference.Parse(imageReference)
if err != nil {
t.Fatalf("failed to parse reference %q: %v", imageReference, err)
}
namedTagged, ok := parsed.(reference.NamedTagged)
if !ok {
t.Fatalf("expected NamedTagged reference, not %T", parsed)
}
imageCount := repoImages[imageReference]
for i := 0; i < imageCount; i++ {
img, err := storeTestImage(ctx, reg, namedTagged, schemaVersion, setManagedByOpenShift)
if err != nil {
t.Fatal(err)
}
arr := result[imageReference]
t.Logf("created image %s@%s image with layers:", namedTagged.Name(), img.Name)
for _, l := range img.DockerImageLayers {
t.Logf(" %s of size %d", l.Name, l.LayerSize)
}
result[imageReference] = append(arr, img)
}
}
return result, nil
}
func newTestRegistry(
ctx context.Context,
osClient client.Interface,
storageDriver driver.StorageDriver,
blobrepositorycachettl time.Duration,
pullthrough bool,
useBlobDescriptorCacheProvider bool,
) (*testRegistry, error) {
if storageDriver == nil {
storageDriver = inmemory.New()
}
dockerStorageDriver = storageDriver
opts := []storage.RegistryOption{
storage.BlobDescriptorServiceFactory(&blobDescriptorServiceFactory{}),
storage.EnableDelete,
storage.EnableRedirect,
}
if useBlobDescriptorCacheProvider {
cacheProvider := cache.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider())
opts = append(opts, storage.BlobDescriptorCacheProvider(cacheProvider))
}
reg, err := storage.NewRegistry(ctx, dockerStorageDriver, opts...)
if err != nil {
return nil, err
}
dockerRegistry = reg
return &testRegistry{
Namespace: dockerRegistry,
osClient: osClient,
blobrepositorycachettl: blobrepositorycachettl,
pullthrough: pullthrough,
}, nil
}
func main() {
var configPath, reposPath string
flag.StringVar(&configPath, "config", "", "path to a config file")
flag.StringVar(&reposPath, "repos", "", "file with a list of repos")
flag.Parse()
if configPath == "" {
fmt.Fprintln(os.Stderr, "must supply a config file with -config")
flag.Usage()
return
}
// Parse config file
configFile, err := os.Open(configPath)
if err != nil {
panic(fmt.Sprintf("error opening config file: %v", err))
}
defer configFile.Close()
config, err := configuration.Parse(configFile)
if err != nil {
panic(fmt.Sprintf("error parsing config file: %v", err))
}
ctx := context.Background()
driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
if err != nil {
panic(fmt.Sprintf("error creating storage driver: %v", err))
}
registry, _ := storage.NewRegistry(ctx, driver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
var repos []string
if reposPath != "" {
reposFile, err := os.Open(reposPath)
if err != nil {
panic(fmt.Sprintf("could not open repos file: %v", err))
}
scanner := bufio.NewScanner(reposFile)
for scanner.Scan() {
repoName := scanner.Text()
if len(repoName) > 0 {
if repoName[0] == '+' {
repoName = repoName[1:]
}
repos = append(repos, repoName)
}
}
} else {
repos = make([]string, maxRepos)
n, err := registry.Repositories(ctx, repos, "")
if err != nil && err != io.EOF {
panic(fmt.Sprintf("unexpected error getting repo: %v", err))
}
if n == maxRepos {
panic("too many repositories")
}
repos = repos[:n]
}
var wg sync.WaitGroup
repoChan := make(chan string)
for i := 0; i < 30; i++ {
wg.Add(1)
go func() {
for repoName := range repoChan {
if err := checkRepo(registry, repoName); err != nil {
fmt.Fprintln(os.Stderr, err)
}
}
wg.Done()
}()
}
for _, repoName := range repos {
repoChan <- repoName
}
close(repoChan)
wg.Wait()
}
// TestAppDispatcher builds an application with a test dispatcher and ensures
// that requests are properly dispatched and the handlers are constructed.
// This only tests the dispatch mechanism. The underlying dispatchers must be
// tested individually.
func TestAppDispatcher(t *testing.T) {
driver := inmemory.New()
ctx := context.Background()
registry, err := storage.NewRegistry(ctx, driver, storage.BlobDescriptorCacheProvider(memorycache.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
app := &App{
Config: &configuration.Configuration{},
Context: ctx,
router: v2.Router(),
driver: driver,
registry: registry,
}
server := httptest.NewServer(app)
router := v2.Router()
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("error parsing server url: %v", err)
}
varCheckingDispatcher := func(expectedVars map[string]string) dispatchFunc {
return func(ctx *Context, r *http.Request) http.Handler {
// Always checks the same name context
if ctx.Repository.Named().Name() != getName(ctx) {
t.Fatalf("unexpected name: %q != %q", ctx.Repository.Named().Name(), "foo/bar")
}
// Check that we have all that is expected
for expectedK, expectedV := range expectedVars {
if ctx.Value(expectedK) != expectedV {
t.Fatalf("unexpected %s in context vars: %q != %q", expectedK, ctx.Value(expectedK), expectedV)
}
}
// Check that we only have variables that are expected
for k, v := range ctx.Value("vars").(map[string]string) {
_, ok := expectedVars[k]
if !ok { // name is checked on context
// We have an unexpected key, fail
t.Fatalf("unexpected key %q in vars with value %q", k, v)
}
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
}
}
// unflatten a list of variables, suitable for gorilla/mux, to a map[string]string
unflatten := func(vars []string) map[string]string {
m := make(map[string]string)
for i := 0; i < len(vars)-1; i = i + 2 {
m[vars[i]] = vars[i+1]
}
return m
}
for _, testcase := range []struct {
endpoint string
vars []string
}{
{
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "foo/bar",
"reference", "sometag",
},
},
{
endpoint: v2.RouteNameTags,
vars: []string{
"name", "foo/bar",
},
},
{
endpoint: v2.RouteNameBlobUpload,
vars: []string{
"name", "foo/bar",
},
},
{
endpoint: v2.RouteNameBlobUploadChunk,
vars: []string{
"name", "foo/bar",
"uuid", "theuuid",
},
},
} {
app.register(testcase.endpoint, varCheckingDispatcher(unflatten(testcase.vars)))
route := router.GetRoute(testcase.endpoint).Host(serverURL.Host)
u, err := route.URL(testcase.vars...)
if err != nil {
t.Fatal(err)
}
resp, err := http.Get(u.String())
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %v != %v", resp.StatusCode, http.StatusOK)
}
}
}
// NewApp takes a configuration and returns a configured app, ready to serve
// requests. The app only implements ServeHTTP and can be wrapped in other
// handlers accordingly.
func NewApp(ctx context.Context, configuration *configuration.Configuration) *App {
app := &App{
Config: configuration,
Context: ctx,
router: v2.RouterWithPrefix(configuration.HTTP.Prefix),
isCache: configuration.Proxy.RemoteURL != "",
}
// Register the handler dispatchers.
app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler {
return http.HandlerFunc(apiBase)
})
app.register(v2.RouteNameManifest, imageManifestDispatcher)
app.register(v2.RouteNameCatalog, catalogDispatcher)
app.register(v2.RouteNameTags, tagsDispatcher)
app.register(v2.RouteNameBlob, blobDispatcher)
app.register(v2.RouteNameBlobUpload, blobUploadDispatcher)
app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher)
var err error
app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters())
if err != nil {
// TODO(stevvooe): Move the creation of a service into a protected
// method, where this is created lazily. Its status can be queried via
// a health check.
panic(err)
}
purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := configuration.Storage["maintenance"]; ok {
if v, ok := mc["uploadpurging"]; ok {
purgeConfig, ok = v.(map[interface{}]interface{})
if !ok {
panic("uploadpurging config key must contain additional keys")
}
}
if v, ok := mc["readonly"]; ok {
readOnly, ok := v.(map[interface{}]interface{})
if !ok {
panic("readonly config key must contain additional keys")
}
if readOnlyEnabled, ok := readOnly["enabled"]; ok {
app.readOnly, ok = readOnlyEnabled.(bool)
if !ok {
panic("readonly's enabled config key must have a boolean value")
}
}
}
}
startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig)
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
if err != nil {
panic(err)
}
app.configureSecret(configuration)
app.configureEvents(configuration)
app.configureRedis(configuration)
app.configureLogHook(configuration)
// Generate an ephemeral key to be used for signing converted manifests
// for clients that don't support schema2.
app.trustKey, err = libtrust.GenerateECP256PrivateKey()
if err != nil {
panic(err)
}
if configuration.HTTP.Host != "" {
u, err := url.Parse(configuration.HTTP.Host)
if err != nil {
panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err))
}
app.httpHost = *u
}
options := []storage.RegistryOption{}
if app.isCache {
options = append(options, storage.DisableDigestResumption)
}
// configure deletion
if d, ok := configuration.Storage["delete"]; ok {
e, ok := d["enabled"]
if ok {
if deleteEnabled, ok := e.(bool); ok && deleteEnabled {
options = append(options, storage.EnableDelete)
}
}
}
// configure redirects
var redirectDisabled bool
if redirectConfig, ok := configuration.Storage["redirect"]; ok {
v := redirectConfig["disable"]
switch v := v.(type) {
case bool:
redirectDisabled = v
default:
panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig))
}
}
if redirectDisabled {
ctxu.GetLogger(app).Infof("backend redirection disabled")
} else {
options = append(options, storage.EnableRedirect)
}
// configure storage caches
if cc, ok := configuration.Storage["cache"]; ok {
v, ok := cc["blobdescriptor"]
if !ok {
// Backwards compatible: "layerinfo" == "blobdescriptor"
v = cc["layerinfo"]
}
switch v {
case "redis":
if app.redis == nil {
panic("redis configuration required to use for layerinfo cache")
}
cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis)
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using redis blob descriptor cache")
case "inmemory":
cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider()
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache")
default:
if v != "" {
ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"])
}
}
}
if app.registry == nil {
// configure the registry if no cache section is available.
app.registry, err = storage.NewRegistry(app.Context, app.driver, options...)
if err != nil {
panic("could not create registry: " + err.Error())
}
}
app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"])
if err != nil {
panic(err)
}
authType := configuration.Auth.Type()
if authType != "" {
accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters())
if err != nil {
panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err))
}
app.accessController = accessController
ctxu.GetLogger(app).Debugf("configured %q access controller", authType)
}
// configure as a pull through cache
if configuration.Proxy.RemoteURL != "" {
app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy)
if err != nil {
panic(err.Error())
}
app.isCache = true
ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL)
}
return app
}
func cmdPush(c *cli.Context) {
url := fmt.Sprintf("http://%s/deploy/upload", c.GlobalString("api-endpoint"))
var payload struct {
DeployKey string `json:"deploy_key"`
}
payload.DeployKey = c.String("deploy-key")
var response struct {
AccessKey string `json:"access_key_id"`
SecretKey string `json:"secret_access_key"`
Bucket string `json:"bucket"`
Region string `json:"region"`
Name string `json:"name"`
}
_, err := napping.Post(url, &payload, &response, nil)
if err != nil {
log.Panic("mikro-cli: push: upload-request: ", err)
}
options := s3.DriverParameters{
AccessKey: response.AccessKey,
SecretKey: response.SecretKey,
Bucket: response.Bucket,
Region: aws.GetRegion(fmt.Sprint(response.Region)),
ChunkSize: 5 << 21,
Secure: true,
}
ctx := context.Background()
driver, err := s3.New(options)
registry, err := storage.NewRegistry(ctx, driver)
imageName := c.Args()[0]
repo, tag := parsers.ParseRepositoryTag(imageName)
fmt.Printf("repo: %s tag: %s\n", repo, tag)
bundle, err := NewImageBundle(repo, tag)
if err != nil {
log.Panic("mikro-cli: push: cannot export image: ", err)
}
defer bundle.Close()
privateKey, err := libtrust.GenerateECP256PrivateKey()
dgst, err := pushImageToRegistry(ctx, response.Name,
c.String("commit-sha"),
repo, tag,
bundle,
privateKey,
registry)
fmt.Printf("Pushed image %s as %s@%s\n", imageName, response.Name, dgst)
// FIXME: we want to use the dgst here, but we can't since ECS doesn't
// support it yet.
registerCommit(c, c.String("deploy-key"), c.String("commit-sha"),
fmt.Sprintf("private:%s:%s", response.Name,
c.String("commit-sha")))
}
// Populate remote store and record the digests
func makeTestEnv(t *testing.T, name string) *testEnv {
nameRef, err := reference.ParseNamed(name)
if err != nil {
t.Fatalf("unable to parse reference: %s", err)
}
ctx := context.Background()
truthDir, err := ioutil.TempDir("", "truth")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
cacheDir, err := ioutil.TempDir("", "cache")
if err != nil {
t.Fatalf("unable to create tempdir: %s", err)
}
localDriver, err := filesystem.FromParameters(map[string]interface{}{
"rootdirectory": truthDir,
})
if err != nil {
t.Fatalf("unable to create filesystem driver: %s", err)
}
// todo: create a tempfile area here
localRegistry, err := storage.NewRegistry(ctx, localDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableRedirect, storage.DisableDigestResumption)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
localRepo, err := localRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
cacheDriver, err := filesystem.FromParameters(map[string]interface{}{
"rootdirectory": cacheDir,
})
if err != nil {
t.Fatalf("unable to create filesystem driver: %s", err)
}
truthRegistry, err := storage.NewRegistry(ctx, cacheDriver, storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()))
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
truthRepo, err := truthRegistry.Repository(ctx, nameRef)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
truthBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: truthRepo.Blobs(ctx),
}
localBlobs := statsBlobStore{
stats: make(map[string]int),
blobs: localRepo.Blobs(ctx),
}
s := scheduler.New(ctx, inmemory.New(), "/scheduler-state.json")
proxyBlobStore := proxyBlobStore{
repositoryName: nameRef,
remoteStore: truthBlobs,
localStore: localBlobs,
scheduler: s,
authChallenger: &mockChallenger{},
}
te := &testEnv{
store: proxyBlobStore,
ctx: ctx,
}
return te
}
// NewApp takes a configuration and returns a configured app, ready to serve
// requests. The app only implements ServeHTTP and can be wrapped in other
// handlers accordingly.
func NewApp(ctx context.Context, configuration configuration.Configuration) *App {
app := &App{
Config: configuration,
Context: ctx,
router: v2.RouterWithPrefix(configuration.HTTP.Prefix),
isCache: configuration.Proxy.RemoteURL != "",
}
app.Context = ctxu.WithLogger(app.Context, ctxu.GetLogger(app, "instance.id"))
// Register the handler dispatchers.
app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler {
return http.HandlerFunc(apiBase)
})
app.register(v2.RouteNameManifest, imageManifestDispatcher)
app.register(v2.RouteNameCatalog, catalogDispatcher)
app.register(v2.RouteNameTags, tagsDispatcher)
app.register(v2.RouteNameBlob, blobDispatcher)
app.register(v2.RouteNameBlobUpload, blobUploadDispatcher)
app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher)
fmt.Printf("Starting to create factory\n")
var err error
app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters())
if err != nil {
// TODO(stevvooe): Move the creation of a service into a protected
// method, where this is created lazily. Its status can be queried via
// a health check.
panic(err)
}
fmt.Printf("Created storage factory\n")
purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := configuration.Storage["maintenance"]; ok {
for k, v := range mc {
switch k {
case "uploadpurging":
purgeConfig = v.(map[interface{}]interface{})
}
}
}
fmt.Printf("Starting uploaduperger\n")
startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig)
fmt.Printf("Starting applyStorageMiddleware\n")
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
if err != nil {
panic(err)
}
fmt.Printf("Starting configurations\n")
app.configureSecret(&configuration)
app.configureEvents(&configuration)
app.configureRedis(&configuration)
app.configureLogHook(&configuration)
options := []storage.RegistryOption{}
if app.isCache {
options = append(options, storage.DisableDigestResumption)
}
// configure deletion
if d, ok := configuration.Storage["delete"]; ok {
e, ok := d["enabled"]
if ok {
if deleteEnabled, ok := e.(bool); ok && deleteEnabled {
options = append(options, storage.EnableDelete)
}
}
}
// configure redirects
var redirectDisabled bool
if redirectConfig, ok := configuration.Storage["redirect"]; ok {
v := redirectConfig["disable"]
switch v := v.(type) {
case bool:
redirectDisabled = v
default:
panic(fmt.Sprintf("invalid type for redirect config: %#v", redirectConfig))
}
}
if redirectDisabled {
ctxu.GetLogger(app).Infof("backend redirection disabled")
} else {
options = append(options, storage.EnableRedirect)
}
fmt.Printf("Configuring caches\n")
// configure storage caches
if cc, ok := configuration.Storage["cache"]; ok {
v, ok := cc["blobdescriptor"]
if !ok {
// Backwards compatible: "layerinfo" == "blobdescriptor"
v = cc["layerinfo"]
}
switch v {
case "redis":
if app.redis == nil {
panic("redis configuration required to use for layerinfo cache")
}
cacheProvider := rediscache.NewRedisBlobDescriptorCacheProvider(app.redis)
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using redis blob descriptor cache")
case "inmemory":
cacheProvider := memorycache.NewInMemoryBlobDescriptorCacheProvider()
localOptions := append(options, storage.BlobDescriptorCacheProvider(cacheProvider))
app.registry, err = storage.NewRegistry(app, app.driver, localOptions...)
if err != nil {
panic("could not create registry: " + err.Error())
}
ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache")
default:
if v != "" {
ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"])
}
}
}
if app.registry == nil {
// configure the registry if no cache section is available.
app.registry, err = storage.NewRegistry(app.Context, app.driver, options...)
if err != nil {
panic("could not create registry: " + err.Error())
}
}
fmt.Printf("Starting applyRegistryMiddleware\n")
app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"])
if err != nil {
panic(err)
}
authType := configuration.Auth.Type()
if authType != "" {
accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters())
if err != nil {
panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err))
}
app.accessController = accessController
ctxu.GetLogger(app).Debugf("configured %q access controller", authType)
}
// configure as a pull through cache
if configuration.Proxy.RemoteURL != "" {
app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy)
if err != nil {
panic(err.Error())
}
app.isCache = true
ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL)
}
fmt.Printf("Created app\n")
return app
}
fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
os.Exit(1)
}
ctx := context.Background()
ctx, err = configureLogging(ctx, config)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
os.Exit(1)
}
k, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
registry, err := storage.NewRegistry(ctx, driver, storage.Schema1SigningKey(k))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
os.Exit(1)
}
err = storage.MarkAndSweep(ctx, driver, registry, dryRun)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
os.Exit(1)
}
},
}
fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
os.Exit(1)
}
ctx := context.Background()
ctx, err = configureLogging(ctx, config)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
os.Exit(1)
}
k, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
os.Exit(1)
}
err = storage.MarkAndSweep(ctx, driver, registry, dryRun)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
os.Exit(1)
}
},
}
func markAndSweep(storageDriver driver.StorageDriver) error {
ctx := context.Background()
// Construct a registry
registry, err := storage.NewRegistry(ctx, storageDriver)
if err != nil {
return fmt.Errorf("failed to construct registry: %v", err)
}
repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
if !ok {
return fmt.Errorf("coercion error: unable to convert Namespace to RepositoryEnumerator")
}
// mark
markSet := make(map[digest.Digest]struct{})
err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
var err error
named, err := reference.ParseNamed(repoName)
if err != nil {
return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
}
repository, err := registry.Repository(ctx, named)
if err != nil {
return fmt.Errorf("failed to construct repository: %v", err)
}
manifestService, err := repository.Manifests(ctx)
if err != nil {
return fmt.Errorf("failed to construct manifest service: %v", err)
}
manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
if !ok {
return fmt.Errorf("coercion error: unable to convert ManifestService into ManifestEnumerator")
}
err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
// Mark the manifest's blob
markSet[dgst] = struct{}{}
manifest, err := manifestService.Get(ctx, dgst)
if err != nil {
return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
}
descriptors := manifest.References()
for _, descriptor := range descriptors {
markSet[descriptor.Digest] = struct{}{}
}
switch manifest.(type) {
case *schema1.SignedManifest:
signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
if !ok {
return fmt.Errorf("coercion error: unable to convert ManifestSErvice into SignaturesGetter")
}
signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
if err != nil {
return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
}
for _, signatureDigest := range signatures {
markSet[signatureDigest] = struct{}{}
}
break
case *schema2.DeserializedManifest:
config := manifest.(*schema2.DeserializedManifest).Config
markSet[config.Digest] = struct{}{}
break
}
return nil
})
return err
})
if err != nil {
return fmt.Errorf("failed to mark: %v\n", err)
}
// sweep
blobService := registry.Blobs()
deleteSet := make(map[digest.Digest]struct{})
err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
// check if digest is in markSet. If not, delete it!
if _, ok := markSet[dgst]; !ok {
deleteSet[dgst] = struct{}{}
}
return nil
})
// Construct vacuum
vacuum := storage.NewVacuum(ctx, storageDriver)
for dgst := range deleteSet {
err = vacuum.RemoveBlob(string(dgst))
if err != nil {
return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
}
}
return err
}