func (m Mux) ListBlobs() ([]string, error) {
ret := make([]string, 0)
for _, e := range m {
blobLister, ok := e.BlobStore.(BlobLister)
if !ok {
return nil, fmt.Errorf("Backend blobstore \"%s\" don't support ListBlobs()", util.TryGetImplName(e.BlobStore))
}
entries, err := blobLister.ListBlobs()
if err != nil {
return nil, fmt.Errorf("Backend blobstore \"%s\" failed to ListBlobs: %v", util.TryGetImplName(e.BlobStore), err)
}
ret = append(ret, entries...)
}
return ret, nil
}
func (cbs *CachedBlobStore) ListBlobs() ([]string, error) {
belister, ok := cbs.backendbs.(blobstore.BlobLister)
if !ok {
return nil, fmt.Errorf("Backendbs \"%v\" doesn't support listing blobs.", util.TryGetImplName(cbs.backendbs))
}
belist, err := belister.ListBlobs()
if err != nil {
return nil, fmt.Errorf("Backendbs failed to ListBlobs: %v", err)
}
cachelist := cbs.entriesmgr.ListBlobs()
cachelistset := make(map[string]struct{})
for _, blobpath := range cachelist {
cachelistset[blobpath] = struct{}{}
}
// list = append(cachelist, ...belist but entries already in cachelist)
list := cachelist
for _, blobpath := range belist {
if _, ok := cachelistset[blobpath]; ok {
// already in cachelist
continue
}
list = append(list, blobpath)
}
return list, nil
}
func Install(srv *mgmt.Server, s *scheduler.Scheduler, bbs blobstore.BlobStore, cbs *cachedblobstore.CachedBlobStore) {
rtr := srv.APIRouter().PathPrefix("/blobstore").Subrouter()
rtr.HandleFunc("/config", mgmt.JSONHandler(func(req *http.Request) interface{} {
type Config struct {
Flags string `json:"flags"`
BackendImplName string `json:"backend_impl_name"`
CacheImplName string `json:"cache_impl_name"`
}
return Config{
Flags: flags.FlagsToString(cbs.Flags()),
BackendImplName: util.TryGetImplName(bbs),
CacheImplName: util.TryGetImplName(cbs),
}
}))
rtr.HandleFunc("/entries", mgmt.JSONHandler(func(req *http.Request) interface{} {
return cbs.DumpEntriesInfo()
}))
rtr.HandleFunc("/reduce_cache", func(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
http.Error(w, "reduce_cache should be triggered with POST method.", http.StatusMethodNotAllowed)
return
}
dryrunp := req.URL.Query().Get("dryrun")
dryrun := len(dryrunp) > 0
desiredSizeP := req.URL.Query().Get("to")
desiredSize, err := humanize.ParseBytes(desiredSizeP)
if desiredSize > math.MaxInt64 || err != nil {
http.Error(w, "Invalid desired size given.", http.StatusInternalServerError)
return
}
jv := s.RunImmediatelyBlock(&cachedblobstore.ReduceCacheTask{cbs, int64(desiredSize), dryrun})
if err := jv.Result.Err(); err != nil {
http.Error(w, "Reduce cache task failed with error", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain")
w.Write([]byte("ok"))
})
}
func Install(srv *mgmt.Server, bbs blobstore.BlobStore, cbs *cachedblobstore.CachedBlobStore) {
rtr := srv.APIRouter().PathPrefix("/blobstore").Subrouter()
rtr.HandleFunc("/config", mgmt.JSONHandler(func(req *http.Request) interface{} {
type Config struct {
Flags string `json:"flags"`
BackendImplName string `json:"backend_impl_name"`
CacheImplName string `json:"cache_impl_name"`
}
return Config{
Flags: flags.FlagsToString(cbs.Flags()),
BackendImplName: util.TryGetImplName(bbs),
CacheImplName: util.TryGetImplName(cbs),
}
}))
rtr.HandleFunc("/entries", mgmt.JSONHandler(func(req *http.Request) interface{} {
return cbs.DumpEntriesInfo()
}))
}
func (m Mux) Open(blobpath string, flags int) (BlobHandle, error) {
bs := m.findBlobStoreFor(blobpath)
if bs == nil {
return nil, ErrEmptyMux
}
rabs, ok := bs.(RandomAccessBlobStore)
if !ok {
return nil, fmt.Errorf("Backend blobstore \"%s\" don't support Open()", util.TryGetImplName(bs))
}
return rabs.Open(blobpath, flags)
}
func (m Mux) RemoveBlob(blobpath string) error {
bs := m.findBlobStoreFor(blobpath)
if bs == nil {
return ErrEmptyMux
}
remover, ok := bs.(BlobRemover)
if !ok {
return fmt.Errorf("Backend blobstore \"%s\" don't support RemoveBlob()", util.TryGetImplName(bs))
}
return remover.RemoveBlob(blobpath)
}
func (m Mux) BlobSize(blobpath string) (int64, error) {
bs := m.findBlobStoreFor(blobpath)
if bs == nil {
return -1, ErrEmptyMux
}
sizer, ok := bs.(BlobSizer)
if !ok {
return -1, fmt.Errorf("Backend blobstore \"%s\" don't support BlobSize()", util.TryGetImplName(bs))
}
return sizer.BlobSize(blobpath)
}
func (cbs *CachedBlobStore) RemoveBlob(blobpath string) error {
backendrm, ok := cbs.backendbs.(blobstore.BlobRemover)
if !ok {
return fmt.Errorf("Backendbs \"%v\" doesn't support removing blobs.", util.TryGetImplName(cbs.backendbs))
}
cacherm, ok := cbs.cachebs.(blobstore.BlobRemover)
if !ok {
return fmt.Errorf("Cachebs \"%v\" doesn't support removing blobs.", util.TryGetImplName(cbs.cachebs))
}
if err := cbs.entriesmgr.RemoveBlob(blobpath); err != nil {
return err
}
cbs.bever.Delete(blobpath)
if err := backendrm.RemoveBlob(blobpath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("Backendbs RemoveBlob failed: %v", err)
}
if err := cacherm.RemoveBlob(blobpath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("Cachebs RemoveBlob failed: %v", err)
}
return nil
}
func (cbs *CachedBlobStore) ReduceCache(ctx context.Context, desiredSize int64, dryrun bool) error {
start := time.Now()
tsizer, ok := cbs.cachebs.(blobstore.TotalSizer)
if !ok {
return fmt.Errorf("Cache backend \"%s\" doesn't support TotalSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
}
blobsizer, ok := cbs.cachebs.(blobstore.BlobSizer)
if !ok {
return fmt.Errorf("Cache backend \"%s\" doesn't support BlobSize() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
}
blobremover, ok := cbs.cachebs.(blobstore.BlobRemover)
if !ok {
return fmt.Errorf("Cache backend \"%s\" doesn't support RemoveBlob() method, required to ReduceCache(). aborting.", util.TryGetImplName(cbs.cachebs))
}
totalSizeBefore, err := tsizer.TotalSize()
if err != nil {
return fmt.Errorf("Failed to query current total cache size: %v", err)
}
needsReduce := totalSizeBefore - desiredSize
if needsReduce < 0 {
logger.Infof(mylog, "ReduceCache: No need to reduce cache as its already under desired size! No-op.")
return nil
}
logger.Infof(mylog, "ReduceCache: Current cache bs total size: %s. Desired size: %s. Needs to reduce %s.",
humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(desiredSize)), humanize.IBytes(uint64(needsReduce)))
bps := cbs.usagestats.FindLeastUsed()
for _, bp := range bps {
size, err := blobsizer.BlobSize(bp)
if err != nil {
if os.IsNotExist(err) {
logger.Infof(mylog, "Attempted to drop blob cache \"%s\", but not found. Maybe it's already removed.", bp)
continue
}
return fmt.Errorf("Failed to query size for cache blob \"%s\": %v", bp, err)
}
logger.Infof(mylog, "ReduceCache: Drop entry \"%s\" to release %s", bp, humanize.IBytes(uint64(size)))
if !dryrun {
if err := cbs.entriesmgr.DropCacheEntry(bp, blobremover); err != nil {
return fmt.Errorf("Failed to remove cache blob \"%s\": %v", bp, err)
}
}
needsReduce -= size
if needsReduce < 0 {
break
}
}
totalSizeAfter, err := tsizer.TotalSize()
if err != nil {
return fmt.Errorf("Failed to query current total cache size: %v", err)
}
logger.Infof(mylog, "ReduceCache done. Cache bs total size: %s -> %s. Dryrun: %t. Took: %s",
humanize.IBytes(uint64(totalSizeBefore)), humanize.IBytes(uint64(totalSizeAfter)),
dryrun, time.Since(start))
return nil
}