示例#1
0
// handle tries to respond to client request by loading metadata and file parts
// from the cache. If there are missing parts, they are retrieved from the upstream.
func (h *reqHandler) handle() {
	h.Logger.Debugf("[%p] Caching proxy access: %s %s", h.req, h.req.Method, h.req.RequestURI)

	rng := h.req.Header.Get("Range")
	obj, err := h.Cache.Storage.GetMetadata(h.objID)
	if os.IsNotExist(err) {
		h.Logger.Debugf("[%p] No metadata on storage, proxying...", h.req)
		h.carbonCopyProxy()
	} else if err != nil {
		h.Logger.Errorf("[%p] Storage error when reading metadata: %s", h.req, err)
		if isTooManyFiles(err) {
			http.Error(
				h.resp,
				http.StatusText(http.StatusInternalServerError),
				http.StatusInternalServerError)
			return
		}
		if discardErr := h.Cache.Storage.Discard(h.objID); discardErr != nil {
			h.Logger.Errorf("[%p] Storage error when discarding of object's data: %s", h.req, discardErr)
		}
		h.carbonCopyProxy()
	} else if !utils.IsMetadataFresh(obj) {
		h.Logger.Debugf("[%p] Metadata is stale, proxying...", h.req)
		//!TODO: optimize, do only a head request when the metadata is stale?
		if discardErr := h.Cache.Storage.Discard(h.objID); discardErr != nil {
			h.Logger.Errorf("[%p] Storage error when discarding of object's data: %s", h.req, discardErr)
		}
		h.carbonCopyProxy()
	} else if !cacheutils.CacheSatisfiesRequest(obj, h.req) {
		h.Logger.Debugf("[%p] Client does not want cached response or the cache does not satisfy the request, proxying...", h.req)
		h.carbonCopyProxy()
	} else {
		h.obj = obj
		//!TODO: advertise that we support ranges - send "Accept-Ranges: bytes"?

		//!TODO: evaluate conditional requests: https://tools.ietf.org/html/rfc7232
		//!TODO: Also, handle this from RFC7233:
		// "The Range header field is evaluated after evaluating the precondition
		// header fields defined in [RFC7232], and only if the result in absence
		// of the Range header field would be a 200 (OK) response.  In other
		// words, Range is ignored when a conditional GET would result in a 304
		// (Not Modified) response."

		if rng != "" {
			h.Logger.Debugf("[%p] Serving range '%s', preferably from cache...", h.req, rng)
			h.knownRanged()
		} else {
			h.Logger.Debugf("[%p] Serving full object, preferably from cache...", h.req)
			h.knownFull()
		}
	}
}
示例#2
0
文件: init.go 项目: gowebabc/nedomi
func (a *Application) reloadCache(cz *types.CacheZone) {
	counter := 0
	callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
		counter++
		//!TODO: remove hardcoded periods and timeout, get them from config
		if counter%100 == 0 {
			select {
			case <-a.ctx.Done():
				return false
			case <-time.After(100 * time.Millisecond):
			}
		}

		if !utils.IsMetadataFresh(obj) {
			if err := cz.Storage.Discard(obj.ID); err != nil {
				a.logger.Errorf("Error on discarding objID `%s` in reloadCache: %s", obj.ID, err)
			}
		} else {
			cz.Scheduler.AddEvent(
				obj.ID.Hash(),
				storage.GetExpirationHandler(cz, a.logger, obj.ID),
				//!TODO: Maybe do not use time.Now but cached time. See the todo comment
				// in utils.IsMetadataFresh.
				time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
			)

			for _, idx := range parts {
				if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
					a.logger.Errorf("Error on adding objID `%s` in reloadCache: %s", obj.ID, err)
				}
			}
		}

		return true
	}

	go func() {
		if err := cz.Storage.Iterate(callback); err != nil {
			a.logger.Errorf("Received iterator error '%s' after loading %d objects", err, counter)
		} else {
			a.logger.Logf("Loading contents from disk finished: %d objects loaded!", counter)
		}
	}()
}
示例#3
0
func (a *Application) reloadCache(cz *types.CacheZone) {
	counter := 0
	callback := func(obj *types.ObjectMetadata, parts ...*types.ObjectIndex) bool {
		counter++
		//!TODO: remove hardcoded periods and timeout, get them from config
		if counter%100 == 0 {
			select {
			case <-a.ctx.Done():
				return false
			case <-time.After(100 * time.Millisecond):
			}
		}

		if !utils.IsMetadataFresh(obj) {
			if err := cz.Storage.Discard(obj.ID); err != nil {
				a.logger.Errorf("Error for cache zone `%s` on discarding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
			}
		} else {
			cz.Scheduler.AddEvent(
				obj.ID.Hash(),
				storage.GetExpirationHandler(cz, a.logger, obj.ID),
				//!TODO: Maybe do not use time.Now but cached time. See the todo comment
				// in utils.IsMetadataFresh.
				time.Unix(obj.ExpiresAt, 0).Sub(time.Now()),
			)

			for _, idx := range parts {
				if err := cz.Algorithm.AddObject(idx); err != nil && err != types.ErrAlreadyInCache {
					a.logger.Errorf("Error for cache zone `%s` on adding objID `%s` in reloadCache: %s", cz.ID, obj.ID, err)
				}
			}
		}

		return true
	}

	go func() {
		var ch = make(chan struct{})
		defer close(ch)
		go func() {
			const tick = 10 * time.Second

			var ticker = time.NewTicker(tick)
			var ticks int64
			defer ticker.Stop()
			for {
				select {
				case <-ticker.C:
					ticks++
					a.logger.Logf("Storage reload for cache zone `%s` has reloaded %d for %s and is still going", cz.ID, counter, time.Duration(ticks)*tick)
				case <-ch:
					return
				}
			}
		}()
		a.logger.Logf("Start storage reload for cache zone `%s`", cz.ID)
		if err := cz.Storage.Iterate(callback); err != nil {
			a.logger.Errorf("For cache zone `%s` received iterator error '%s' after loading %d objects", cz.ID, err, counter)
		} else {
			a.logger.Logf("Loading contents from disk for cache zone `%s` finished: %d objects loaded!", cz.ID, counter)
		}
	}()
}