Example #1
0
func (m Mux) ListBlobs() ([]string, error) {
	ret := make([]string, 0)
	for _, e := range m {
		blobLister, ok := e.BlobStore.(BlobLister)
		if !ok {
			return nil, fmt.Errorf("Backend blobstore \"%s\" don't support ListBlobs()", util.TryGetImplName(e.BlobStore))
		}
		entries, err := blobLister.ListBlobs()
		if err != nil {
			return nil, fmt.Errorf("Backend blobstore \"%s\" failed to ListBlobs: %v", util.TryGetImplName(e.BlobStore), err)
		}
		ret = append(ret, entries...)
	}
	return ret, nil
}
Example #2
0
func (cbs *CachedBlobStore) ListBlobs() ([]string, error) {
	belister, ok := cbs.backendbs.(blobstore.BlobLister)
	if !ok {
		return nil, fmt.Errorf("Backendbs \"%v\" doesn't support listing blobs.", util.TryGetImplName(cbs.backendbs))
	}

	belist, err := belister.ListBlobs()
	if err != nil {
		return nil, fmt.Errorf("Backendbs failed to ListBlobs: %v", err)
	}
	cachelist := cbs.entriesmgr.ListBlobs()
	cachelistset := make(map[string]struct{})
	for _, blobpath := range cachelist {
		cachelistset[blobpath] = struct{}{}
	}

	// list = append(cachelist, ...belist but entries already in cachelist)
	list := cachelist
	for _, blobpath := range belist {
		if _, ok := cachelistset[blobpath]; ok {
			// already in cachelist
			continue
		}
		list = append(list, blobpath)
	}

	return list, nil
}
Example #3
0
func Install(srv *mgmt.Server, s *scheduler.Scheduler, bbs blobstore.BlobStore, cbs *cachedblobstore.CachedBlobStore) {
	rtr := srv.APIRouter().PathPrefix("/blobstore").Subrouter()

	rtr.HandleFunc("/config", mgmt.JSONHandler(func(req *http.Request) interface{} {
		type Config struct {
			Flags           string `json:"flags"`
			BackendImplName string `json:"backend_impl_name"`
			CacheImplName   string `json:"cache_impl_name"`
		}
		return Config{
			Flags:           flags.FlagsToString(cbs.Flags()),
			BackendImplName: util.TryGetImplName(bbs),
			CacheImplName:   util.TryGetImplName(cbs),
		}
	}))
	rtr.HandleFunc("/entries", mgmt.JSONHandler(func(req *http.Request) interface{} {
		return cbs.DumpEntriesInfo()
	}))
	rtr.HandleFunc("/reduce_cache", func(w http.ResponseWriter, req *http.Request) {
		if req.Method != "POST" {
			http.Error(w, "reduce_cache should be triggered with POST method.", http.StatusMethodNotAllowed)
			return
		}

		dryrunp := req.URL.Query().Get("dryrun")
		dryrun := len(dryrunp) > 0

		desiredSizeP := req.URL.Query().Get("to")
		desiredSize, err := humanize.ParseBytes(desiredSizeP)
		if desiredSize > math.MaxInt64 || err != nil {
			http.Error(w, "Invalid desired size given.", http.StatusInternalServerError)
			return
		}

		jv := s.RunImmediatelyBlock(&cachedblobstore.ReduceCacheTask{cbs, int64(desiredSize), dryrun})
		if err := jv.Result.Err(); err != nil {
			http.Error(w, "Reduce cache task failed with error", http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte("ok"))
	})
}
Example #4
0
func Install(srv *mgmt.Server, bbs blobstore.BlobStore, cbs *cachedblobstore.CachedBlobStore) {
	rtr := srv.APIRouter().PathPrefix("/blobstore").Subrouter()

	rtr.HandleFunc("/config", mgmt.JSONHandler(func(req *http.Request) interface{} {
		type Config struct {
			Flags           string `json:"flags"`
			BackendImplName string `json:"backend_impl_name"`
			CacheImplName   string `json:"cache_impl_name"`
		}
		return Config{
			Flags:           flags.FlagsToString(cbs.Flags()),
			BackendImplName: util.TryGetImplName(bbs),
			CacheImplName:   util.TryGetImplName(cbs),
		}
	}))
	rtr.HandleFunc("/entries", mgmt.JSONHandler(func(req *http.Request) interface{} {
		return cbs.DumpEntriesInfo()
	}))
}
Example #5
0
func (m Mux) Open(blobpath string, flags int) (BlobHandle, error) {
	bs := m.findBlobStoreFor(blobpath)
	if bs == nil {
		return nil, ErrEmptyMux
	}
	rabs, ok := bs.(RandomAccessBlobStore)
	if !ok {
		return nil, fmt.Errorf("Backend blobstore \"%s\" don't support Open()", util.TryGetImplName(bs))
	}
	return rabs.Open(blobpath, flags)
}
Example #6
0
func (m Mux) RemoveBlob(blobpath string) error {
	bs := m.findBlobStoreFor(blobpath)
	if bs == nil {
		return ErrEmptyMux
	}
	remover, ok := bs.(BlobRemover)
	if !ok {
		return fmt.Errorf("Backend blobstore \"%s\" don't support RemoveBlob()", util.TryGetImplName(bs))
	}
	return remover.RemoveBlob(blobpath)
}
Example #7
0
func (m Mux) BlobSize(blobpath string) (int64, error) {
	bs := m.findBlobStoreFor(blobpath)
	if bs == nil {
		return -1, ErrEmptyMux
	}
	sizer, ok := bs.(BlobSizer)
	if !ok {
		return -1, fmt.Errorf("Backend blobstore \"%s\" don't support BlobSize()", util.TryGetImplName(bs))
	}
	return sizer.BlobSize(blobpath)
}
Example #8
0
func (cbs *CachedBlobStore) RemoveBlob(blobpath string) error {
	backendrm, ok := cbs.backendbs.(blobstore.BlobRemover)
	if !ok {
		return fmt.Errorf("Backendbs \"%v\" doesn't support removing blobs.", util.TryGetImplName(cbs.backendbs))
	}
	cacherm, ok := cbs.cachebs.(blobstore.BlobRemover)
	if !ok {
		return fmt.Errorf("Cachebs \"%v\" doesn't support removing blobs.", util.TryGetImplName(cbs.cachebs))
	}

	if err := cbs.entriesmgr.RemoveBlob(blobpath); err != nil {
		return err
	}
	cbs.bever.Delete(blobpath)
	if err := backendrm.RemoveBlob(blobpath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Backendbs RemoveBlob failed: %v", err)
	}
	if err := cacherm.RemoveBlob(blobpath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Cachebs RemoveBlob failed: %v", err)
	}

	return nil
}
Example #9
0
func (cbs *CachedBlobStore) ReduceCache(ctx context.Context, desiredSize int64, dryrun bool) error {
	start := time.Now()

	tsizer, ok := cbs.cachebs.(blobstore.TotalSizer)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support TotalSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	blobsizer, ok := cbs.cachebs.(blobstore.BlobSizer)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support BlobSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	blobremover, ok := cbs.cachebs.(blobstore.BlobRemover)
	if !ok {
		return fmt.Errorf("Cache backend \"%s\" doesn't support RemoveBlob() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
	}

	totalSizeBefore, err := tsizer.TotalSize()
	if err != nil {
		return fmt.Errorf("Failed to query current total cache size: %v", err)
	}

	needsReduce := totalSizeBefore - desiredSize
	if needsReduce < 0 {
		logger.Infof(mylog, "ReduceCache: No need to reduce cache as its already under desired size! No-op.")
		return nil
	}
	logger.Infof(mylog, "ReduceCache: Current cache bs total size: %s. Desired size: %s. Needs to reduce %s.",
		humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(desiredSize)), humanize.IBytes(uint64(needsReduce)))

	bps := cbs.usagestats.FindLeastUsed()
	for _, bp := range bps {
		size, err := blobsizer.BlobSize(bp)
		if err != nil {
			if os.IsNotExist(err) {
				logger.Infof(mylog, "Attempted to drop blob cache \"%s\", but not found. Maybe it's already removed.", bp)
				continue
			}
			return fmt.Errorf("Failed to query size for cache blob \"%s\": %v", bp, err)
		}

		logger.Infof(mylog, "ReduceCache: Drop entry \"%s\" to release %s", bp, humanize.IBytes(uint64(size)))

		if !dryrun {
			if err := cbs.entriesmgr.DropCacheEntry(bp, blobremover); err != nil {
				return fmt.Errorf("Failed to remove cache blob \"%s\": %v", bp, err)
			}
		}

		needsReduce -= size
		if needsReduce < 0 {
			break
		}
	}

	totalSizeAfter, err := tsizer.TotalSize()
	if err != nil {
		return fmt.Errorf("Failed to query current total cache size: %v", err)
	}

	logger.Infof(mylog, "ReduceCache done. Cache bs total size: %s -> %s. Dryrun: %t. Took: %s",
		humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(totalSizeAfter)),
		dryrun, time.Since(start))
	return nil
}