// handle tries to respond to client request by loading metadata and file parts
// from the cache. If there are missing parts, they are retrieved from the upstream.
func (h *reqHandler) handle() {
h.Logger.Debugf("[%p] Caching proxy access: %s %s", h.req, h.req.Method, h.req.RequestURI)
rng := h.req.Header.Get("Range")
obj, err := h.Cache.Storage.GetMetadata(h.objID)
if os.IsNotExist(err) {
h.Logger.Debugf("[%p] No metadata on storage, proxying...", h.req)
h.carbonCopyProxy()
} else if err != nil {
h.Logger.Errorf("[%p] Storage error when reading metadata: %s", h.req, err)
if isTooManyFiles(err) {
http.Error(
h.resp,
http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
return
}
if discardErr := h.Cache.Storage.Discard(h.objID); discardErr != nil {
h.Logger.Errorf("[%p] Storage error when discarding of object's data: %s", h.req, discardErr)
}
h.carbonCopyProxy()
} else if !utils.IsMetadataFresh(obj) {
h.Logger.Debugf("[%p] Metadata is stale, proxying...", h.req)
//!TODO: optimize, do only a head request when the metadata is stale?
if discardErr := h.Cache.Storage.Discard(h.objID); discardErr != nil {
h.Logger.Errorf("[%p] Storage error when discarding of object's data: %s", h.req, discardErr)
}
h.carbonCopyProxy()
} else if !cacheutils.CacheSatisfiesRequest(obj, h.req) {
h.Logger.Debugf("[%p] Client does not want cached response or the cache does not satisfy the request, proxying...", h.req)
h.carbonCopyProxy()
} else {
h.obj = obj
//!TODO: advertise that we support ranges - send "Accept-Ranges: bytes"?
//!TODO: evaluate conditional requests: https://tools.ietf.org/html/rfc7232
//!TODO: Also, handle this from RFC7233:
// "The Range header field is evaluated after evaluating the precondition
// header fields defined in [RFC7232], and only if the result in absence
// of the Range header field would be a 200 (OK) response. In other
// words, Range is ignored when a conditional GET would result in a 304
// (Not Modified) response."
if rng != "" {
h.Logger.Debugf("[%p] Serving range '%s', preferably from cache...", h.req, rng)
h.knownRanged()
} else {
h.Logger.Debugf("[%p] Serving full object, preferably from cache...", h.req)
h.knownFull()
}
}
}
func (a *Application) reloadCache(cz *types.CacheZone) {
counter := 0
callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
counter++
//!TODO: remove hardcoded periods and timeout, get them from config
if counter%100 == 0 {
select {
case <-a.ctx.Done():
return false
case <-time.After(100 * time.Millisecond):
}
}
if !utils.IsMetadataFresh(obj) {
if err := cz.Storage.Discard(obj.ID); err != nil {
a.logger.Errorf("Error on discarding objID `%s` in reloadCache: %s", obj.ID, err)
}
} else {
cz.Scheduler.AddEvent(
obj.ID.Hash(),
storage.GetExpirationHandler(cz, a.logger, obj.ID),
//!TODO: Maybe do not use time.Now but cached time. See the todo comment
// in utils.IsMetadataFresh.
time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
)
for _, idx := range parts {
if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
a.logger.Errorf("Error on adding objID `%s` in reloadCache: %s", obj.ID, err)
}
}
}
return true
}
go func() {
if err := cz.Storage.Iterate(callback); err != nil {
a.logger.Errorf("Received iterator error '%s' after loading %d objects", err, counter)
} else {
a.logger.Logf("Loading contents from disk finished: %d objects loaded!", counter)
}
}()
}
func (a *Application) reloadCache(cz *types.CacheZone) {
counter := 0
callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
counter++
//!TODO: remove hardcoded periods and timeout, get them from config
if counter%100 == 0 {
select {
case <-a.ctx.Done():
return false
case <-time.After(100 * time.Millisecond):
}
}
if !utils.IsMetadataFresh(obj) {
if err := cz.Storage.Discard(obj.ID); err != nil {
a.logger.Errorf("Error for cache zone `%s` on discarding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
}
} else {
cz.Scheduler.AddEvent(
obj.ID.Hash(),
storage.GetExpirationHandler(cz, a.logger, obj.ID),
//!TODO: Maybe do not use time.Now but cached time. See the todo comment
// in utils.IsMetadataFresh.
time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
)
for _, idx := range parts {
if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
a.logger.Errorf("Error for cache zone `%s` on adding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
}
}
}
return true
}
go func() {
var ch = make(chan struct{})
defer close(ch)
go func() {
const tick = 10 * time.Second
var ticker = time.NewTicker(tick)
var ticks int64
defer ticker.Stop()
for {
select {
case <-ticker.C:
ticks++
a.logger.Logf("Storage reload for cache zone `%s` has reloaded %d for %s and is still going", cz.ID, counter, time.Duration(ticks)*tick)
case <-ch:
return
}
}
}()
a.logger.Logf("Start storage reload for cache zone `%s`", cz.ID)
if err := cz.Storage.Iterate(callback); err != nil {
a.logger.Errorf("For cache zone `%s` received iterator error '%s' after loading %d objects", cz.ID, err, counter)
} else {
a.logger.Logf("Loading contents from disk for cache zone `%s` finished: %d objects loaded!", cz.ID, counter)
}
}()
}