func (a *Application) reloadCache(cz *types.CacheZone) {
counter := 0
callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
counter++
//!TODO: remove hardcoded periods and timeout, get them from config
if counter%100 == 0 {
select {
case <-a.ctx.Done():
return false
case <-time.After(100 * time.Millisecond):
}
}
if !utils.IsMetadataFresh(obj) {
if err := cz.Storage.Discard(obj.ID); err != nil {
a.logger.Errorf("Error on discarding objID `%s` in reloadCache: %s", obj.ID, err)
}
} else {
cz.Scheduler.AddEvent(
obj.ID.Hash(),
storage.GetExpirationHandler(cz, a.logger, obj.ID),
//!TODO: Maybe do not use time.Now but cached time. See the todo comment
// in utils.IsMetadataFresh.
time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
)
for _, idx := range parts {
if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
a.logger.Errorf("Error on adding objID `%s` in reloadCache: %s", obj.ID, err)
}
}
}
return true
}
go func() {
if err := cz.Storage.Iterate(callback); err != nil {
a.logger.Errorf("Received iterator error '%s' after loading %d objects", err, counter)
} else {
a.logger.Logf("Loading contents from disk finished: %d objects loaded!", counter)
}
}()
}
func (h *reqHandler) getResponseHook() func(*httputils.FlexibleResponseWriter) {
return func(rw *httputils.FlexibleResponseWriter) {
h.Logger.Debugf("[%p] Received headers for %s, sending them to client...", h.req, h.req.URL)
httputils.CopyHeadersWithout(rw.Headers, h.resp.Header(), hopHeaders...)
h.resp.WriteHeader(rw.Code)
isCacheable := cacheutils.IsResponseCacheable(rw.Code, rw.Headers)
if !isCacheable {
h.Logger.Debugf("[%p] Response is non-cacheable", h.req)
rw.BodyWriter = utils.AddCloser(h.resp)
return
}
expiresIn := cacheutils.ResponseExpiresIn(rw.Headers, h.CacheDefaultDuration)
if expiresIn <= 0 {
h.Logger.Debugf("[%p] Response expires in the past: %s", h.req, expiresIn)
rw.BodyWriter = utils.AddCloser(h.resp)
return
}
responseRange, err := httputils.GetResponseRange(rw.Code, rw.Headers)
if err != nil {
h.Logger.Debugf("[%p] Was not able to get response range (%s)", h.req, err)
rw.BodyWriter = utils.AddCloser(h.resp)
return
}
h.Logger.Debugf("[%p] Response is cacheable! Caching metadata and parts...", h.req)
code := rw.Code
if code == http.StatusPartialContent {
// 206 is returned only if the server would have returned 200 with a normal request
code = http.StatusOK
}
//!TODO: maybe call cached time.Now. See the comment in utils.IsMetadataFresh
now := time.Now()
obj := &types.ObjectMetadata{
ID: h.objID,
ResponseTimestamp: now.Unix(),
Code: code,
Size: responseRange.ObjSize,
Headers: make(http.Header),
ExpiresAt: now.Add(expiresIn).Unix(),
}
httputils.CopyHeadersWithout(rw.Headers, obj.Headers, metadataHeadersToFilter...)
if obj.Headers.Get("Date") == "" { // maybe the server does not return date, we should set it then
obj.Headers.Set("Date", now.Format(http.TimeFormat))
}
//!TODO: consult the cache algorithm whether to save the metadata
//!TODO: optimize this, save the metadata only when it's newer
//!TODO: also, error if we already have fresh metadata but the received metadata is different
if err := h.Cache.Storage.SaveMetadata(obj); err != nil {
h.Logger.Errorf("[%p] Could not save metadata for %s: %s", h.req, obj.ID, err)
rw.BodyWriter = utils.AddCloser(h.resp)
return
}
if h.req.Method == "HEAD" {
rw.BodyWriter = utils.AddCloser(h.resp)
return
}
rw.BodyWriter = utils.MultiWriteCloser(
utils.AddCloser(h.resp),
PartWriter(h.Cache, h.objID, *responseRange),
)
h.Logger.Debugf("[%p] Setting the cached data to expire in %s", h.req, expiresIn)
h.Cache.Scheduler.AddEvent(
h.objID.Hash(),
storage.GetExpirationHandler(h.Cache, h.Logger, h.objID),
expiresIn,
)
}
}
func (a *Application) reloadCache(cz *types.CacheZone) {
counter := 0
callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
counter++
//!TODO: remove hardcoded periods and timeout, get them from config
if counter%100 == 0 {
select {
case <-a.ctx.Done():
return false
case <-time.After(100 * time.Millisecond):
}
}
if !utils.IsMetadataFresh(obj) {
if err := cz.Storage.Discard(obj.ID); err != nil {
a.logger.Errorf("Error for cache zone `%s` on discarding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
}
} else {
cz.Scheduler.AddEvent(
obj.ID.Hash(),
storage.GetExpirationHandler(cz, a.logger, obj.ID),
//!TODO: Maybe do not use time.Now but cached time. See the todo comment
// in utils.IsMetadataFresh.
time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
)
for _, idx := range parts {
if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
a.logger.Errorf("Error for cache zone `%s` on adding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
}
}
}
return true
}
go func() {
var ch = make(chan struct{})
defer close(ch)
go func() {
const tick = 10 * time.Second
var ticker = time.NewTicker(tick)
var ticks int64
defer ticker.Stop()
for {
select {
case <-ticker.C:
ticks++
a.logger.Logf("Storage reload for cache zone `%s` has reloaded %d for %s and is still going", cz.ID, counter, time.Duration(ticks)*tick)
case <-ch:
return
}
}
}()
a.logger.Logf("Start storage reload for cache zone `%s`", cz.ID)
if err := cz.Storage.Iterate(callback); err != nil {
a.logger.Errorf("For cache zone `%s` received iterator error '%s' after loading %d objects", cz.ID, err, counter)
} else {
a.logger.Logf("Loading contents from disk for cache zone `%s` finished: %d objects loaded!", cz.ID, counter)
}
}()
}