Example #1
0
func (a *Application) reloadCache(cz *types.CacheZone) {
	counter := 0
	callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
		counter++
		//!TODO: remove hardcoded periods and timeout, get them from config
		if counter%100 == 0 {
			select {
			case <-a.ctx.Done():
				return false
			case <-time.After(100 * time.Millisecond):
			}
		}

		if !utils.IsMetadataFresh(obj) {
			if err := cz.Storage.Discard(obj.ID); err != nil {
				a.logger.Errorf("Error on discarding objID `%s` in reloadCache: %s", obj.ID, err)
			}
		} else {
			cz.Scheduler.AddEvent(
				obj.ID.Hash(),
				storage.GetExpirationHandler(cz, a.logger, obj.ID),
				//!TODO: Maybe do not use time.Now but cached time. See the todo comment
				// in utils.IsMetadataFresh.
				time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
			)

			for _, idx := range parts {
				if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
					a.logger.Errorf("Error on adding objID `%s` in reloadCache: %s", obj.ID, err)
				}
			}
		}

		return true
	}

	go func() {
		if err := cz.Storage.Iterate(callback); err != nil {
			a.logger.Errorf("Received iterator error '%s' after loading %d objects", err, counter)
		} else {
			a.logger.Logf("Loading contents from disk finished: %d objects loaded!", counter)
		}
	}()
}
func (h *reqHandler) getResponseHook() func(*httputils.FlexibleResponseWriter) {

	return func(rw *httputils.FlexibleResponseWriter) {
		h.Logger.Debugf("[%p] Received headers for %s, sending them to client...", h.req, h.req.URL)
		httputils.CopyHeadersWithout(rw.Headers, h.resp.Header(), hopHeaders...)
		h.resp.WriteHeader(rw.Code)

		isCacheable := cacheutils.IsResponseCacheable(rw.Code, rw.Headers)
		if !isCacheable {
			h.Logger.Debugf("[%p] Response is non-cacheable", h.req)
			rw.BodyWriter = utils.AddCloser(h.resp)
			return
		}

		expiresIn := cacheutils.ResponseExpiresIn(rw.Headers, h.CacheDefaultDuration)
		if expiresIn <= 0 {
			h.Logger.Debugf("[%p] Response expires in the past: %s", h.req, expiresIn)
			rw.BodyWriter = utils.AddCloser(h.resp)
			return
		}

		responseRange, err := httputils.GetResponseRange(rw.Code, rw.Headers)
		if err != nil {
			h.Logger.Debugf("[%p] Was not able to get response range (%s)", h.req, err)
			rw.BodyWriter = utils.AddCloser(h.resp)
			return
		}

		h.Logger.Debugf("[%p] Response is cacheable! Caching metadata and parts...", h.req)

		code := rw.Code
		if code == http.StatusPartialContent {
			// 206 is returned only if the server would have returned 200 with a normal request
			code = http.StatusOK
		}

		//!TODO: maybe call cached time.Now. See the comment in utils.IsMetadataFresh
		now := time.Now()

		obj := &types.ObjectMetadata{
			ID:                h.objID,
			ResponseTimestamp: now.Unix(),
			Code:              code,
			Size:              responseRange.ObjSize,
			Headers:           make(http.Header),
			ExpiresAt:         now.Add(expiresIn).Unix(),
		}
		httputils.CopyHeadersWithout(rw.Headers, obj.Headers, metadataHeadersToFilter...)
		if obj.Headers.Get("Date") == "" { // maybe the server does not return date, we should set it then
			obj.Headers.Set("Date", now.Format(http.TimeFormat))
		}

		//!TODO: consult the cache algorithm whether to save the metadata
		//!TODO: optimize this, save the metadata only when it's newer
		//!TODO: also, error if we already have fresh metadata but the received metadata is different
		if err := h.Cache.Storage.SaveMetadata(obj); err != nil {
			h.Logger.Errorf("[%p] Could not save metadata for %s: %s", h.req, obj.ID, err)
			rw.BodyWriter = utils.AddCloser(h.resp)
			return
		}

		if h.req.Method == "HEAD" {
			rw.BodyWriter = utils.AddCloser(h.resp)
			return
		}

		rw.BodyWriter = utils.MultiWriteCloser(
			utils.AddCloser(h.resp),
			PartWriter(h.Cache, h.objID, *responseRange),
		)

		h.Logger.Debugf("[%p] Setting the cached data to expire in %s", h.req, expiresIn)
		h.Cache.Scheduler.AddEvent(
			h.objID.Hash(),
			storage.GetExpirationHandler(h.Cache, h.Logger, h.objID),
			expiresIn,
		)
	}
}
Example #3
0
func (a *Application) reloadCache(cz *types.CacheZone) {
	counter := 0
	callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
		counter++
		//!TODO: remove hardcoded periods and timeout, get them from config
		if counter%100 == 0 {
			select {
			case <-a.ctx.Done():
				return false
			case <-time.After(100 * time.Millisecond):
			}
		}

		if !utils.IsMetadataFresh(obj) {
			if err := cz.Storage.Discard(obj.ID); err != nil {
				a.logger.Errorf("Error for cache zone `%s` on discarding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
			}
		} else {
			cz.Scheduler.AddEvent(
				obj.ID.Hash(),
				storage.GetExpirationHandler(cz, a.logger, obj.ID),
				//!TODO: Maybe do not use time.Now but cached time. See the todo comment
				// in utils.IsMetadataFresh.
				time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
			)

			for _, idx := range parts {
				if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
					a.logger.Errorf("Error for cache zone `%s` on adding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
				}
			}
		}

		return true
	}

	go func() {
		var ch = make(chan struct{})
		defer close(ch)
		go func() {
			const tick = 10 * time.Second

			var ticker = time.NewTicker(tick)
			var ticks int64
			defer ticker.Stop()
			for {
				select {
				case <-ticker.C:
					ticks++
					a.logger.Logf("Storage reload for cache zone `%s` has reloaded %d for %s and is still going", cz.ID, counter, time.Duration(ticks)*tick)
				case <-ch:
					return
				}
			}
		}()
		a.logger.Logf("Start storage reload for cache zone `%s`", cz.ID)
		if err := cz.Storage.Iterate(callback); err != nil {
			a.logger.Errorf("For cache zone `%s` received iterator error '%s' after loading %d objects", cz.ID, err, counter)
		} else {
			a.logger.Logf("Loading contents from disk for cache zone `%s` finished: %d objects loaded!", cz.ID, counter)
		}
	}()
}