// SwapLimiter safely swaps current limiter for this queue with the passed one if capacities or qps's differ.
func (q *RateLimitedTimedQueue) SwapLimiter(newQPS float32) {
q.limiterLock.Lock()
defer q.limiterLock.Unlock()
if q.limiter.QPS() == newQPS {
return
}
var newLimiter flowcontrol.RateLimiter
if newQPS <= 0 {
newLimiter = flowcontrol.NewFakeNeverRateLimiter()
} else {
newLimiter = flowcontrol.NewTokenBucketRateLimiter(newQPS, evictionRateLimiterBurst)
}
// If we're currently waiting on limiter, we drain the new one - this is a good approach when Burst value is 1
// TODO: figure out if we need to support higher Burst values and decide on the drain logic, should we keep:
// - saturation (percentage of used tokens)
// - number of used tokens
// - number of available tokens
// - something else
for q.limiter.Saturation() > newLimiter.Saturation() {
// Check if we're not using fake limiter
previousSaturation := newLimiter.Saturation()
newLimiter.TryAccept()
// It's a fake limiter
if newLimiter.Saturation() == previousSaturation {
break
}
}
q.limiter.Stop()
q.limiter = newLimiter
}
// RegisterMetricAndTrackRateLimiterUsage registers a metric ownerName_rate_limiter_use in prometheus to track
// how much used rateLimiter is and starts a goroutine that updates this metric every updatePeriod
func RegisterMetricAndTrackRateLimiterUsage(ownerName string, rateLimiter flowcontrol.RateLimiter) error {
err := registerRateLimiterMetric(ownerName)
if err != nil {
return err
}
go wait.Forever(func() {
metricsLock.Lock()
defer metricsLock.Unlock()
rateLimiterMetrics[ownerName].Set(rateLimiter.Saturation())
}, updatePeriod)
return nil
}
func importRepositoryFromDockerV1(ctx gocontext.Context, repository *importRepository, limiter flowcontrol.RateLimiter) {
value := ctx.Value(ContextKeyV1RegistryClient)
if value == nil {
err := kapierrors.NewForbidden(api.Resource(""), "", fmt.Errorf("registry %q does not support the v2 Registry API", repository.Registry.Host))
err.ErrStatus.Reason = "NotV2Registry"
applyErrorToRepository(repository, err)
return
}
client, ok := value.(dockerregistry.Client)
if !ok {
err := kapierrors.NewForbidden(api.Resource(""), "", fmt.Errorf("registry %q does not support the v2 Registry API", repository.Registry.Host))
err.ErrStatus.Reason = "NotV2Registry"
return
}
conn, err := client.Connect(repository.Registry.Host, repository.Insecure)
if err != nil {
applyErrorToRepository(repository, err)
return
}
// if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N
if count := repository.MaximumTags; count > 0 {
tagMap, err := conn.ImageTags(repository.Ref.Namespace, repository.Ref.Name)
if err != nil {
repository.Err = err
return
}
tags := make([]string, 0, len(tagMap))
for tag := range tagMap {
tags = append(tags, tag)
}
// some images on the Hub have empty tags - treat those as "latest"
set := sets.NewString(tags...)
if set.Has("") {
set.Delete("")
set.Insert(api.DefaultImageTag)
}
tags = set.List()
// include only the top N tags in the result, put the rest in AdditionalTags
api.PrioritizeTags(tags)
for _, s := range tags {
if count <= 0 {
repository.AdditionalTags = append(repository.AdditionalTags, s)
continue
}
count--
repository.Tags = append(repository.Tags, importTag{
Name: s,
})
}
}
// load digests
for i := range repository.Digests {
importDigest := &repository.Digests[i]
if importDigest.Err != nil || importDigest.Image != nil {
continue
}
limiter.Accept()
image, err := conn.ImageByID(repository.Ref.Namespace, repository.Ref.Name, importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
// we do not preserve manifests of legacy images
importDigest.Image, err = schema0ToImage(image, importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
}
for i := range repository.Tags {
importTag := &repository.Tags[i]
if importTag.Err != nil || importTag.Image != nil {
continue
}
limiter.Accept()
image, err := conn.ImageByTag(repository.Ref.Namespace, repository.Ref.Name, importTag.Name)
if err != nil {
importTag.Err = err
continue
}
// we do not preserve manifests of legacy images
importTag.Image, err = schema0ToImage(image, "")
if err != nil {
importTag.Err = err
continue
}
}
}
// importRepositoryFromDocker loads the tags and images requested in the passed importRepository, obeying the
// optional rate limiter. Errors are set onto the individual tags and digest objects.
func (isi *ImageStreamImporter) importRepositoryFromDocker(ctx gocontext.Context, retriever RepositoryRetriever, repository *importRepository, limiter flowcontrol.RateLimiter) {
glog.V(5).Infof("importing remote Docker repository registry=%s repository=%s insecure=%t", repository.Registry, repository.Name, repository.Insecure)
// retrieve the repository
repo, err := retriever.Repository(ctx, repository.Registry, repository.Name, repository.Insecure)
if err != nil {
glog.V(5).Infof("unable to access repository %#v: %#v", repository, err)
switch {
case err == reference.ErrReferenceInvalidFormat:
err = field.Invalid(field.NewPath("from", "name"), repository.Name, "the provided repository name is not valid")
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.Contains(err.Error(), "tls: oversized record received with length") && !repository.Insecure:
err = kapierrors.NewBadRequest("this repository is HTTP only and requires the insecure flag to import")
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "does not support v2 API"):
importRepositoryFromDockerV1(ctx, repository, limiter)
return
}
applyErrorToRepository(repository, err)
return
}
// get a manifest context
s, err := repo.Manifests(ctx)
if err != nil {
glog.V(5).Infof("unable to access manifests for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
case strings.HasSuffix(err.Error(), "no basic auth credentials"):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q and did not have credentials to the repository", repository.Ref.Exact()))
}
applyErrorToRepository(repository, err)
return
}
// get a blob context
b := repo.Blobs(ctx)
// if repository import is requested (MaximumTags), attempt to load the tags, sort them, and request the first N
if count := repository.MaximumTags; count > 0 || count == -1 {
tags, err := repo.Tags(ctx).All(ctx)
if err != nil {
glog.V(5).Infof("unable to access tags for repository %#v: %#v", repository, err)
switch {
case isDockerError(err, v2.ErrorCodeNameUnknown):
err = kapierrors.NewNotFound(api.Resource("dockerimage"), repository.Ref.Exact())
case isDockerError(err, errcode.ErrorCodeUnauthorized):
err = kapierrors.NewUnauthorized(fmt.Sprintf("you may not have access to the Docker image %q", repository.Ref.Exact()))
}
repository.Err = err
return
}
// some images on the Hub have empty tags - treat those as "latest"
set := sets.NewString(tags...)
if set.Has("") {
set.Delete("")
set.Insert(api.DefaultImageTag)
}
tags = set.List()
// include only the top N tags in the result, put the rest in AdditionalTags
api.PrioritizeTags(tags)
for _, s := range tags {
if count <= 0 && repository.MaximumTags != -1 {
repository.AdditionalTags = append(repository.AdditionalTags, s)
continue
}
count--
repository.Tags = append(repository.Tags, importTag{
Name: s,
})
}
}
// load digests
for i := range repository.Digests {
importDigest := &repository.Digests[i]
if importDigest.Err != nil || importDigest.Image != nil {
continue
}
d, err := digest.ParseDigest(importDigest.Name)
if err != nil {
importDigest.Err = err
continue
}
limiter.Accept()
manifest, err := s.Get(ctx, d)
if err != nil {
glog.V(5).Infof("unable to access digest %q for repository %#v: %#v", d, repository, err)
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
continue
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importDigest.Image, err = schema1ToImage(signedManifest, d)
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access the image config using digest %q for repository %#v: %#v", d, repository, err)
if isDockerError(err, v2.ErrorCodeManifestUnknown) {
ref := repository.Ref
ref.ID = deserializedManifest.Config.Digest.String()
importDigest.Err = kapierrors.NewNotFound(api.Resource("dockerimage"), ref.Exact())
} else {
importDigest.Err = formatRepositoryError(repository, "", importDigest.Name, err)
}
continue
}
importDigest.Image, err = schema2ToImage(deserializedManifest, imageConfig, d)
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importDigest.Err = err
continue
}
if err := api.ImageWithMetadata(importDigest.Image); err != nil {
importDigest.Err = err
continue
}
if importDigest.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importDigest.Image); err != nil {
importDigest.Err = err
continue
}
}
}
for i := range repository.Tags {
importTag := &repository.Tags[i]
if importTag.Err != nil || importTag.Image != nil {
continue
}
limiter.Accept()
desc, err := repo.Tags(ctx).Get(ctx, importTag.Name)
if err != nil {
glog.V(5).Infof("unable to get tag %q for repository %#v: %#v", importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
manifest, err := s.Get(ctx, desc.Digest)
if err != nil {
glog.V(5).Infof("unable to access digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
if signedManifest, isSchema1 := manifest.(*schema1.SignedManifest); isSchema1 {
importTag.Image, err = schema1ToImage(signedManifest, "")
} else if deserializedManifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 {
imageConfig, err := b.Get(ctx, deserializedManifest.Config.Digest)
if err != nil {
glog.V(5).Infof("unable to access image config using digest %q for tag %q for repository %#v: %#v", desc.Digest, importTag.Name, repository, err)
importTag.Err = formatRepositoryError(repository, importTag.Name, "", err)
continue
}
importTag.Image, err = schema2ToImage(deserializedManifest, imageConfig, "")
} else {
glog.V(5).Infof("unsupported manifest type: %T", manifest)
continue
}
if err != nil {
importTag.Err = err
continue
}
if err := api.ImageWithMetadata(importTag.Image); err != nil {
importTag.Err = err
continue
}
if importTag.Image.DockerImageMetadata.Size == 0 {
if err := isi.calculateImageSize(ctx, repo, importTag.Image); err != nil {
importTag.Err = err
continue
}
}
}
}