func (a *Application) reloadCache(cz *types.CacheZone) { counter := 0 callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool { counter++ //!TODO: remove hardcoded periods and timeout, get them from config if counter%100 == 0 { select { case <-a.ctx.Done(): return false case <-time.After(100 * time.Millisecond): } } if !utils.IsMetadataFresh(obj) { if err := cz.Storage.Discard(obj.ID); err != nil { a.logger.Errorf("Error on discarding objID `%s` in reloadCache: %s", obj.ID, err) } } else { cz.Scheduler.AddEvent( obj.ID.Hash(), storage.GetExpirationHandler(cz, a.logger, obj.ID), //!TODO: Maybe do not use time.Now but cached time. See the todo comment // in utils.IsMetadataFresh. time.Unix(obj.ExpiresAt, 0).Sub(time.Now()), ) for _, idx := range parts { if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache { a.logger.Errorf("Error on adding objID `%s` in reloadCache: %s", obj.ID, err) } } } return true } go func() { if err := cz.Storage.Iterate(callback); err != nil { a.logger.Errorf("Received iterator error '%s' after loading %d objects", err, counter) } else { a.logger.Logf("Loading contents from disk finished: %d objects loaded!", counter) } }() }
func (h *reqHandler) getResponseHook() func(*httputils.FlexibleResponseWriter) { return func(rw *httputils.FlexibleResponseWriter) { h.Logger.Debugf("[%p] Received headers for %s, sending them to client...", h.req, h.req.URL) httputils.CopyHeadersWithout(rw.Headers, h.resp.Header(), hopHeaders...) h.resp.WriteHeader(rw.Code) isCacheable := cacheutils.IsResponseCacheable(rw.Code, rw.Headers) if !isCacheable { h.Logger.Debugf("[%p] Response is non-cacheable", h.req) rw.BodyWriter = utils.AddCloser(h.resp) return } expiresIn := cacheutils.ResponseExpiresIn(rw.Headers, h.CacheDefaultDuration) if expiresIn <= 0 { h.Logger.Debugf("[%p] Response expires in the past: %s", h.req, expiresIn) rw.BodyWriter = utils.AddCloser(h.resp) return } responseRange, err := httputils.GetResponseRange(rw.Code, rw.Headers) if err != nil { h.Logger.Debugf("[%p] Was not able to get response range (%s)", h.req, err) rw.BodyWriter = utils.AddCloser(h.resp) return } h.Logger.Debugf("[%p] Response is cacheable! Caching metadata and parts...", h.req) code := rw.Code if code == http.StatusPartialContent { // 206 is returned only if the server would have returned 200 with a normal request code = http.StatusOK } //!TODO: maybe call cached time.Now. See the comment in utils.IsMetadataFresh now := time.Now() obj := &types.ObjectMetadata{ ID: h.objID, ResponseTimestamp: now.Unix(), Code: code, Size: responseRange.ObjSize, Headers: make(http.Header), ExpiresAt: now.Add(expiresIn).Unix(), } httputils.CopyHeadersWithout(rw.Headers, obj.Headers, metadataHeadersToFilter...) if obj.Headers.Get("Date") == "" { // maybe the server does not return date, we should set it then obj.Headers.Set("Date", now.Format(http.TimeFormat)) } //!TODO: consult the cache algorithm whether to save the metadata //!TODO: optimize this, save the metadata only when it's newer //!TODO: also, error if we already have fresh metadata but the received metadata is different if err := h.Cache.Storage.SaveMetadata(obj); err != nil { h.Logger.Errorf("[%p] Could not save metadata for %s: %s", h.req, obj.ID, err) rw.BodyWriter = utils.AddCloser(h.resp) return } if h.req.Method == "HEAD" { rw.BodyWriter = utils.AddCloser(h.resp) return } rw.BodyWriter = utils.MultiWriteCloser( utils.AddCloser(h.resp), PartWriter(h.Cache, h.objID, *responseRange), ) h.Logger.Debugf("[%p] Setting the cached data to expire in %s", h.req, expiresIn) h.Cache.Scheduler.AddEvent( h.objID.Hash(), storage.GetExpirationHandler(h.Cache, h.Logger, h.objID), expiresIn, ) } }
func (a *Application) reloadCache(cz *types.CacheZone) { counter := 0 callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool { counter++ //!TODO: remove hardcoded periods and timeout, get them from config if counter%100 == 0 { select { case <-a.ctx.Done(): return false case <-time.After(100 * time.Millisecond): } } if !utils.IsMetadataFresh(obj) { if err := cz.Storage.Discard(obj.ID); err != nil { a.logger.Errorf("Error for cache zone `%s` on discarding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err) } } else { cz.Scheduler.AddEvent( obj.ID.Hash(), storage.GetExpirationHandler(cz, a.logger, obj.ID), //!TODO: Maybe do not use time.Now but cached time. See the todo comment // in utils.IsMetadataFresh. time.Unix(obj.ExpiresAt, 0).Sub(time.Now()), ) for _, idx := range parts { if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache { a.logger.Errorf("Error for cache zone `%s` on adding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err) } } } return true } go func() { var ch = make(chan struct{}) defer close(ch) go func() { const tick = 10 * time.Second var ticker = time.NewTicker(tick) var ticks int64 defer ticker.Stop() for { select { case <-ticker.C: ticks++ a.logger.Logf("Storage reload for cache zone `%s` has reloaded %d for %s and is still going", cz.ID, counter, time.Duration(ticks)*tick) case <-ch: return } } }() a.logger.Logf("Start storage reload for cache zone `%s`", cz.ID) if err := cz.Storage.Iterate(callback); err != nil { a.logger.Errorf("For cache zone `%s` received iterator error '%s' after loading %d objects", cz.ID, err, counter) } else { a.logger.Logf("Loading contents from disk for cache zone `%s` finished: %d objects loaded!", cz.ID, counter) } }() }